diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68eaca4ed3a2a944e32510b744c9294bf0e4d417..3786a35f70ef06e31dd87d4a33d493555974cc0e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,14 +24,14 @@ variables: GIT_SUBMODULE_STRATEGY: recursive # base image -image: gekkofs/core:0.9.2 +image: gekkofs/core:0.9.3-exp ################################################################################ ## Validating ################################################################################ check format: stage: lint - image: gekkofs/linter:0.9.2 + image: gekkofs/linter:0.9.3-exp needs: [] script: - ${SCRIPTS_DIR}/check_format.sh @@ -45,7 +45,7 @@ check format: ################################################################################ gkfs: stage: build - image: gekkofs/deps:0.9.2 + image: gekkofs/deps:0.9.3-exp interruptible: true needs: [] script: @@ -67,7 +67,7 @@ gkfs: gkfwd: stage: build - image: gekkofs/deps:0.9.2 + image: gekkofs/deps:0.9.3-exp interruptible: true needs: [] script: @@ -89,7 +89,7 @@ gkfwd: ## == tests for scripts ==================== scripts: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3-exp needs: [] script: - mkdir -p ${BUILD_PATH}/tests/scripts @@ -105,7 +105,7 @@ scripts: ## == integration tests for gkfs =========== gkfs:integration: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3-exp interruptible: true needs: ['gkfs'] parallel: @@ -154,7 +154,7 @@ gkfs:integration: ## == integration tests for gkfwd ========== gkfwd:integration: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3-exp interruptible: true needs: ['gkfwd'] parallel: @@ -204,7 +204,7 @@ gkfwd:integration: ## == unit tests for gkfs ================== gkfs:unit: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3-exp needs: ['gkfs'] script: ## Add path to mkfs.kreon @@ -242,7 +242,7 @@ gkfs:unit: ################################################################################ documentation: stage: docs - image: gekkofs/docs:0.9.2 + image: gekkofs/docs:0.9.3-exp needs: [] rules: # we only build the documentation automatically if we are on the @@ -272,7 +272,7 @@ documentation: ## == coverage baseline ==================== coverage:baseline: stage: report - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3-exp interruptible: true needs: ['gkfs', 'gkfwd'] @@ -298,7 +298,7 @@ coverage:baseline: coverage: stage: report - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3-exp # needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration', # 'gkfs:unit' ] needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit' ] diff --git a/CHANGELOG.md b/CHANGELOG.md index f360b69b91ab61433facc33aaad5bc321d549b8b..e7b1302c8666b7310e22029d2b46566eb90a0c3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] - +- Replication without using the server. NUM_REPL (0 < NUM_REPL < num_servers) env variable defines the number of +replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)). +- Modified write and reads to use a bitset instead of the traditional hash per chunk in the server. +- Added reattemp support in get_fs_config to other servers, when the initial server fails. +- Added support for Erasure codes ([!168]) using Jerasure lib and adding support for Read error injection. +- Added support for on demand erasure codes calculation with `gkfs_ec_ondemand(fd)` ### New - Additional tests to increase code coverage ([!141](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)). diff --git a/CMake/FindGF_complete.cmake b/CMake/FindGF_complete.cmake new file mode 100644 index 0000000000000000000000000000000000000000..4af6878ea932d475f7345967a67d5c5256c42aca --- /dev/null +++ b/CMake/FindGF_complete.cmake @@ -0,0 +1,68 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# +# - Try to find GF_complete library +# This will define +# GF_complete_FOUND +# GF_complete_INCLUDE_DIR +# GF_complete_LIBRARIES +# + +find_path(GF_complete_INCLUDE_DIR + NAMES gf_complete.h + ) + +find_library(GF_complete_LIBRARY + NAMES gf_complete + ) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( GF_complete + DEFAULT_MSG + GF_complete_INCLUDE_DIR + GF_complete_LIBRARY +) + +if(GF_complete_FOUND) + set(GF_complete_INCLUDE_DIRS ${GF_complete_INCLUDE_DIR}) + set(GF_complete_LIBRARIES ${GF_complete_LIBRARY}) + + if(NOT TARGET GF_complete::GF_complete) + add_library(GF_complete::GF_complete UNKNOWN IMPORTED) + set_target_properties(GF_complete::GF_complete PROPERTIES + IMPORTED_LOCATION "${GF_complete_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${GF_complete_INCLUDE_DIR}" + ) + endif() +endif() + +mark_as_advanced( + GF_complete_INCLUDE_DIR + GF_complete_LIBRARY +) diff --git a/CMake/FindJerasure.cmake b/CMake/FindJerasure.cmake new file mode 100644 index 0000000000000000000000000000000000000000..8117b47e6147de9f1ef32eda02e07f38cc963c39 --- /dev/null +++ b/CMake/FindJerasure.cmake @@ -0,0 +1,68 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# +# - Try to find Jerasure library +# This will define +# Jerasure_FOUND +# Jerasure_INCLUDE_DIR +# Jerasure_LIBRARIES +# + +find_path(Jerasure_INCLUDE_DIR + NAMES jerasure/jerasure.h + ) + +find_library(Jerasure_LIBRARY + NAMES Jerasure + ) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Jerasure + DEFAULT_MSG + Jerasure_INCLUDE_DIR + Jerasure_LIBRARY +) + +if(Jerasure_FOUND) + set(Jerasure_INCLUDE_DIRS ${Jerasure_INCLUDE_DIR}) + set(Jerasure_LIBRARIES ${Jerasure_LIBRARY}) + + if(NOT TARGET Jerasure::Jerasure) + add_library(Jerasure::Jerasure UNKNOWN IMPORTED) + set_target_properties(Jerasure::Jerasure PROPERTIES + IMPORTED_LOCATION "${Jerasure_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${Jerasure_INCLUDE_DIR}" + ) + endif() +endif() + +mark_as_advanced( + Jerasure_INCLUDE_DIR + Jerasure_LIBRARY +) \ No newline at end of file diff --git a/CMake/gkfs-options.cmake b/CMake/gkfs-options.cmake index 598de903ca9a82d9dd4496a2bd96a704792fe2ad..775b9275c00adda90078d0297396b2fbadc47c06 100644 --- a/CMake/gkfs-options.cmake +++ b/CMake/gkfs-options.cmake @@ -370,4 +370,20 @@ gkfs_define_option( DESCRIPTION "If GKFS_ENABLE_FORWARDING is ON, use AGIOS for scheduling I/Os" ) +################################################################################ +# Erasure codes +################################################################################ +gkfs_define_option( + GKFS_ENABLE_EC + HELP_TEXT "Enable Erasure Code" + DEFAULT_VALUE OFF + DESCRIPTION "Use Jerasure for erasure codes reliability" +) + +gkfs_define_option( + GKFS_ENABLE_READ_ERRORS + HELP_TEXT "Enable Read Errors using replication" + DEFAULT_VALUE OFF + DESCRIPTION "Inject read errors" +) diff --git a/CMakeLists.txt b/CMakeLists.txt index 71e7c842f6c73bc814feaff9203b6c97048f8058..d61d8a675ecaffb397b7ffed281137f5308929f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -203,6 +203,18 @@ if(GKFS_ENABLE_PROMETHEUS) find_package(prometheus-cpp REQUIRED) # >= 1.0.0 endif() +### Jerasure: needed for the calculation of Erasure codes +if(GKFS_ENABLE_EC) + message(STATUS "[${PROJECT_NAME}] Checking for Jerasure") + add_compile_definitions(GKFS_ENABLE_EC) + find_package(GF_complete REQUIRED) + find_package(Jerasure REQUIRED) +endif() + +if(GKFS_ENABLE_READ_ERRORS) + add_compile_definitions(GKFS_ENABLE_READ_ERRORS) +endif() + ### Other stuff that can be found out using find_package: # determine the thread library of the system diff --git a/CMakePresets.json b/CMakePresets.json index 88fc9ce80df87c14a5217b3334b7e30e1dd8bfcb..819894f00182724c1a9239aea0a89fcfb56b867f 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -86,7 +86,8 @@ "GKFS_ENABLE_PROMETHEUS": true, "GKFS_RENAME_SUPPORT": true, "MAX_OPEN_FDS": "10000", - "MAX_INTERNAL_FDS": "1024" + "MAX_INTERNAL_FDS": "1024", + "GKFS_ENABLE_EC": true } }, { diff --git a/README.md b/README.md index 6f571eff69a818ce00c1c4ecfb64a9f6876bb9ba..2da692a2a26bfe92d1facb99e8aa426121d4e130 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,24 @@ Support for fstat in renamed files is included. This is disabled by default. +### Replication +The user can enable the data replication feature by setting the replication environment variable: +`LIBGKFS_NUM_REPL=`. +The number of replicas should go from 0 to the number of servers-1. +The replication environment variable can be set up for each client, independently. + +### Erasure codes +The user can enable resilience with erasure codes with `-DGKFS_ENABLE_EC` +Using `LIBGKFS_NUM_REPL=`, the user can define the number of EC servers. +The total servers available for data will be -> total servervs - num_repl + +Additionaly the user can enable the `-DGKFS_ENABLE_READ_ERRORS` to inject a +50% of read errors into the workflow. + +We also included a function to call the EC calculation on demand over a open file: +`gkfs_ec_ondemand(int fd)`, to use it requires the environment variable `LIBGKFS_EC_ONDEMAND=1` + +The function is useful to calculate the ec only when the file is completed, like checkpoints. ## Acknowledgment This software was partially supported by the EC H2020 funded NEXTGenIO project (Project ID: 671951, www.nextgenio.eu). diff --git a/docker/0.9.3-exp/core/Dockerfile b/docker/0.9.3-exp/core/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..a55434924b2cfe720195c371de6736ac049826f7 --- /dev/null +++ b/docker/0.9.3-exp/core/Dockerfile @@ -0,0 +1,40 @@ +FROM debian:bullseye-slim + +LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + curl \ + ca-certificates \ + libtool \ + pkg-config \ + make \ + automake \ + gcc \ + g++ \ + ninja-build \ + procps \ + # AGIOS dependencies + libconfig-dev \ + # Mercury dependencies + libltdl-dev \ + lbzip2 \ + # Margo dependencies \ + libjson-c-dev \ + # RocksDB dependencies + liblz4-dev \ + # syscall_intercept dependencies + libcapstone-dev \ + # GekkoFS dependencies + libboost-program-options-dev \ + uuid-dev && \ + # install cmake 3.14+ since it's needed for some dependencies + curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \ + chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \ + ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean && \ + rm ./cmake-3.25.2-Linux-x86_64.sh diff --git a/docker/0.9.3-exp/core/Makefile b/docker/0.9.3-exp/core/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..e11d6cd4bcf5eb26c09aa45029a4a12fdd0b3364 --- /dev/null +++ b/docker/0.9.3-exp/core/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/core:0.9.3-exp . + +aarch64: + docker build --platform aarch64 -t gekkofs/core:0.9.3-exp . + +all: + docker build -t gekkofs/core:0.9.3-exp . diff --git a/docker/0.9.3-exp/deps/Dockerfile b/docker/0.9.3-exp/deps/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..0ea72dfa9096d155295a59f36412329606fe736c --- /dev/null +++ b/docker/0.9.3-exp/deps/Dockerfile @@ -0,0 +1,38 @@ +FROM gekkofs/core:0.9.3-exp + +LABEL Description="Debian-based environment to build GekkoFS" + +ENV GKFS_PATH /opt/gkfs +ENV GKFS_VERSION 0.9.3-exp + +ENV SCRIPTS_PATH ${GKFS_PATH}/scripts +ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src +ENV INSTALL_PATH /usr/local + +COPY scripts/dl_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/patches ${SCRIPTS_PATH}/patches +COPY scripts/profiles ${SCRIPTS_PATH}/profiles + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + python3-dev \ + python3-venv \ + python3-setuptools \ + libnuma-dev libyaml-dev libcurl4-openssl-dev \ + procps && \ + python3 -m pip install --upgrade pip && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && apt-get autoclean + +# Download and build dependencies +RUN cd ${SCRIPTS_PATH} && \ + /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \ + /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \ + cp ${DEPS_SRC_PATH}/parallax/lib/include/parallax/structures.h ${INSTALL_PATH}/include/ &&\ + rm -rf ${DEPS_SRC_PATH} && \ + rm -rf ${SCRIPTS_PATH} && \ + rmdir ${GKFS_PATH} && \ + ldconfig diff --git a/docker/0.9.3-exp/deps/Makefile b/docker/0.9.3-exp/deps/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..41a10361ee08f8d1b2c242c8e0ca102bb139bb8b --- /dev/null +++ b/docker/0.9.3-exp/deps/Makefile @@ -0,0 +1,23 @@ +CWD:=$(shell pwd) +GIT_ROOT:=$(shell git rev-parse --show-toplevel) + +.PHONY: all build mount-scripts umount-scripts + +all: build mount-scripts remove-scripts +amd64: build-amd64 mount-scripts remove-scripts +aarch64: build-aarch64 mount-scripts remove-scripts + +copy-scripts: + cp -R $(GIT_ROOT)/scripts $(CWD)/scripts + +build: copy-scripts + docker build -t gekkofs/deps:0.9.3-exp . + +build-amd64: copy-scripts + docker build --platform amd64 -t gekkofs/deps:0.9.3-exp . + +build-aarch64: copy-scripts + docker build --platform aarch64 -t gekkofs/deps:0.9.3-exp . + +remove-scripts: + - rm -rf $(CWD)/scripts diff --git a/docker/0.9.3-exp/docs/Dockerfile b/docker/0.9.3-exp/docs/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..55a44fdc1332af58f99c3f46c43cadbdf3ccdf6b --- /dev/null +++ b/docker/0.9.3-exp/docs/Dockerfile @@ -0,0 +1,34 @@ +FROM gekkofs/deps:0.9.3-exp + +LABEL Description="Debian-based environment suitable to build GekkoFS' documentation" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # install dependencies for Doxygen + python \ + flex \ + bison \ + graphviz && \ + # install doxygen (repo version is kind of old) + cd /tmp && curl -L https://sourceforge.net/projects/doxygen/files/rel-1.9.2/doxygen-1.9.2.src.tar.gz/download --output doxygen-1.9.2.src.tar.gz && \ + tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \ + mkdir -p /tmp/doxygen-1.9.2/build && \ + cd /tmp/doxygen-1.9.2/build && \ + cmake -G "Unix Makefiles" .. && \ + make -j8 install && \ + # install sphinx, breathe and exhale + pip3 install \ + 'sphinx==4.4.0' \ + sphinx_rtd_theme \ + 'breathe==4.33.1' \ + 'exhale==0.3.1' \ + 'sphinx-copybutton==0.5.0' \ + 'sphinx-multiversion==0.2.4' \ + 'myst_parser==0.17.0' \ + attrs && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + rm -rf /tmp/doxygen-1.9.2 && \ + rm /tmp/doxygen-1.9.2.src.tar.gz && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.3-exp/docs/Makefile b/docker/0.9.3-exp/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..e7608ea74952aa7565432790c1391a3ae1f2270a --- /dev/null +++ b/docker/0.9.3-exp/docs/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/docs:0.9.3-exp . + +aarch64: + docker build --platform aarch64 -t gekkofs/docs:0.9.3-exp . + +all: + docker build -t gekkofs/docs:0.9.3-exp . \ No newline at end of file diff --git a/docker/0.9.3-exp/linter/Dockerfile b/docker/0.9.3-exp/linter/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..82d87f86d99604d7d725dd2324e7cb9c8690e622 --- /dev/null +++ b/docker/0.9.3-exp/linter/Dockerfile @@ -0,0 +1,19 @@ +FROM gekkofs/core:0.9.3-exp + +LABEL Description="Debian-based environment to check the formatting of GekkoFS code" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # clang 15 deps for clang-format + lsb-release \ + wget \ + software-properties-common \ + gnupg2 && \ + # add clang-15 repos + wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 15 && \ + # install clang-format + apt-get update && apt-get install -y --no-install-recommends clang-format-15 && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.3-exp/linter/Makefile b/docker/0.9.3-exp/linter/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..98f344150a289781171e9e118fb2354b2e1b5f2d --- /dev/null +++ b/docker/0.9.3-exp/linter/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/linter:0.9.3-exp . + +aarch64: + docker build --platform aarch64 -t gekkofs/linter:0.9.3-exp . + +all: + docker build -t gekkofs/linter:0.9.3-exp . diff --git a/docker/0.9.3-exp/testing/Dockerfile b/docker/0.9.3-exp/testing/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..250fea63c48a79e185ad1c40b9b38eeb1cb8c3b6 --- /dev/null +++ b/docker/0.9.3-exp/testing/Dockerfile @@ -0,0 +1,32 @@ +FROM gekkofs/deps:0.9.3-exp + +LABEL Description="Debian-based environment to test GekkoFS" + +RUN \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + # required by lcov's genhtml + libgd-perl && \ + # install cmake 3.21+ since we need to produce JUnit XML files + curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \ + chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \ + ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \ + # install loguru + # (required by several of our scripts) + pip3 install loguru && \ + # install lcov_cobertura + # (required to produce Cobertura XML reports) + pip3 install lcov_cobertura && \ + # install lcov + # (required to produce partial coverage reports in parallel runs) + curl -OL https://github.com/linux-test-project/lcov/releases/download/v1.16/lcov-1.16.tar.gz && \ + tar xfz lcov-1.16.tar.gz && \ + cd lcov-1.16 && \ + make install && \ + cd .. && \ + # cleanup + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && \ + apt-get autoclean && \ + rm ./cmake-3.25.2-Linux-x86_64.sh && \ + rm -rf ./lcov-1.16.* diff --git a/docker/0.9.3-exp/testing/Makefile b/docker/0.9.3-exp/testing/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..26551ad435ada88b301cc52742d501df5ce8a05b --- /dev/null +++ b/docker/0.9.3-exp/testing/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/testing:0.9.3-exp . + +aarch64: + docker build --platform aarch64 -t gekkofs/testing:0.9.3-exp . + +all: + docker build -t gekkofs/testing:0.9.3-exp . diff --git a/include/client/env.hpp b/include/client/env.hpp index c35ae1ad30a857ac037d593dba1cba979c08b787..7f408e5f36d70570e49b0482ca1c16b5cfd04744 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -52,6 +52,10 @@ static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE"); #endif +static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL"); +static constexpr auto EC_ONDEMAND = ADD_PREFIX("EC_ONDEMAND"); + + } // namespace gkfs::env #undef ADD_PREFIX diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index d005d2fb950ff0563553c08b5ea6cd4c112e5693..66e2c755b4fac11b17077b4d95fb8e4f9cbe00f7 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -159,4 +159,10 @@ gkfs_rename(const std::string& old_path, const std::string& new_path); extern "C" int gkfs_getsingleserverdir(const char* path, struct dirent_extended* dirp, unsigned int count, int server); + +#ifdef GKFS_ENABLE_EC +extern "C" int +gkfs_ec_ondemand(const unsigned int fd); +#endif + #endif // GEKKOFS_GKFS_FUNCTIONS_HPP diff --git a/include/client/preload_context.hpp b/include/client/preload_context.hpp index 26875435b3f1546b9dd455160d845811ade38166..9fc4dd1c9a020b0643e5b3edb543257553bb4090 100644 --- a/include/client/preload_context.hpp +++ b/include/client/preload_context.hpp @@ -104,6 +104,8 @@ private: bool internal_fds_must_relocate_; std::bitset protected_fds_; std::string hostname; + int replicas_; + bool ec_ondemand_; public: static PreloadContext* @@ -215,6 +217,18 @@ public: std::string get_hostname(); + + void + set_replicas(const int repl); + + int + get_replicas(); + + bool + get_ec_ondemand(); + + void + set_ec_ondemand(const bool ec_ondemand); }; } // namespace preload diff --git a/include/client/rpc/forward_data.hpp b/include/client/rpc/forward_data.hpp index 8518e997f61de93818c112ec951f44e181713fc1..5248e90bd890c94f8ba921c1cfe9ee7bdceead11 100644 --- a/include/client/rpc/forward_data.hpp +++ b/include/client/rpc/forward_data.hpp @@ -30,6 +30,9 @@ #ifndef GEKKOFS_CLIENT_FORWARD_DATA_HPP #define GEKKOFS_CLIENT_FORWARD_DATA_HPP +#include +#include +#include namespace gkfs::rpc { struct ChunkStat { @@ -41,20 +44,36 @@ struct ChunkStat { // TODO once we have LEAF, remove all the error code returns and throw them as // an exception. +std::pair +ecc_forward_read(const std::string& path, void* buf, const size_t read_size, + const int8_t server); + +std::pair +ecc_forward_write(const std::string& path, const void* buf, + const size_t write_size, const int8_t server); + std::pair forward_write(const std::string& path, const void* buf, off64_t offset, - size_t write_size); + size_t write_size, const int8_t num_copy = 0); std::pair forward_read(const std::string& path, void* buf, off64_t offset, - size_t read_size); + size_t read_size, const int8_t num_copies, + std::set& failed); int -forward_truncate(const std::string& path, size_t current_size, size_t new_size); +forward_truncate(const std::string& path, size_t current_size, size_t new_size, + const int8_t num_copies); std::pair forward_get_chunk_stat(); +std::pair +calc_op_chunks(const std::string& path, const bool append_flag, + const off64_t in_offset, const size_t write_size, + const int64_t updated_metadentry_size); + + } // namespace gkfs::rpc #endif // GEKKOFS_CLIENT_FORWARD_DATA_HPP diff --git a/include/client/rpc/forward_metadata.hpp b/include/client/rpc/forward_metadata.hpp index 564dc903b592ead71947a4696331214a323e685b..97849a5757cfa3718cc80a9c3d24e0d855200860 100644 --- a/include/client/rpc/forward_metadata.hpp +++ b/include/client/rpc/forward_metadata.hpp @@ -50,10 +50,10 @@ class Metadata; namespace rpc { int -forward_create(const std::string& path, mode_t mode); +forward_create(const std::string& path, mode_t mode, const int copy); int -forward_stat(const std::string& path, std::string& attr); +forward_stat(const std::string& path, std::string& attr, const int copy); #ifdef HAS_RENAME int @@ -62,22 +62,24 @@ forward_rename(const std::string& oldpath, const std::string& newpath, #endif // HAS_RENAME int -forward_remove(const std::string& path); +forward_remove(const std::string& path, const int8_t num_copies); int -forward_decr_size(const std::string& path, size_t length); +forward_decr_size(const std::string& path, size_t length, const int copy); int -forward_update_metadentry( - const std::string& path, const gkfs::metadata::Metadata& md, - const gkfs::metadata::MetadentryUpdateFlags& md_flags); +forward_update_metadentry(const std::string& path, + const gkfs::metadata::Metadata& md, + const gkfs::metadata::MetadentryUpdateFlags& md_flags, + const int copy); std::pair forward_update_metadentry_size(const std::string& path, size_t size, - off64_t offset, bool append_flag); + off64_t offset, bool append_flag, + const int num_copies); std::pair -forward_get_metadentry_size(const std::string& path); +forward_get_metadentry_size(const std::string& path, const int copy); std::pair> forward_get_dirents(const std::string& path); diff --git a/include/client/rpc/rpc_types.hpp b/include/client/rpc/rpc_types.hpp index 5993a31c300f0c03bfe55b0eca45eeee94836a38..bc8afd67651afd067b988b2cd0552533d8029835 100644 --- a/include/client/rpc/rpc_types.hpp +++ b/include/client/rpc/rpc_types.hpp @@ -1469,11 +1469,11 @@ struct write_data { public: input(const std::string& path, int64_t offset, uint64_t host_id, - uint64_t host_size, uint64_t chunk_n, uint64_t chunk_start, - uint64_t chunk_end, uint64_t total_chunk_size, - const hermes::exposed_memory& buffers) + uint64_t host_size, const std::string& wbitset, uint64_t chunk_n, + uint64_t chunk_start, uint64_t chunk_end, + uint64_t total_chunk_size, const hermes::exposed_memory& buffers) : m_path(path), m_offset(offset), m_host_id(host_id), - m_host_size(host_size), m_chunk_n(chunk_n), + m_host_size(host_size), m_wbitset(wbitset), m_chunk_n(chunk_n), m_chunk_start(chunk_start), m_chunk_end(chunk_end), m_total_chunk_size(total_chunk_size), m_buffers(buffers) {} @@ -1512,6 +1512,11 @@ struct write_data { return m_chunk_n; } + std::string + wbitset() const { + return m_wbitset; + } + uint64_t chunk_start() const { return m_chunk_start; @@ -1535,15 +1540,16 @@ struct write_data { explicit input(const rpc_write_data_in_t& other) : m_path(other.path), m_offset(other.offset), m_host_id(other.host_id), m_host_size(other.host_size), - m_chunk_n(other.chunk_n), m_chunk_start(other.chunk_start), - m_chunk_end(other.chunk_end), + m_wbitset(other.wbitset), m_chunk_n(other.chunk_n), + m_chunk_start(other.chunk_start), m_chunk_end(other.chunk_end), m_total_chunk_size(other.total_chunk_size), m_buffers(other.bulk_handle) {} explicit operator rpc_write_data_in_t() { - return {m_path.c_str(), m_offset, m_host_id, - m_host_size, m_chunk_n, m_chunk_start, - m_chunk_end, m_total_chunk_size, hg_bulk_t(m_buffers)}; + return {m_path.c_str(), m_offset, m_host_id, + m_host_size, m_wbitset.c_str(), m_chunk_n, + m_chunk_start, m_chunk_end, m_total_chunk_size, + hg_bulk_t(m_buffers)}; } private: @@ -1551,6 +1557,7 @@ struct write_data { int64_t m_offset; uint64_t m_host_id; uint64_t m_host_size; + std::string m_wbitset; uint64_t m_chunk_n; uint64_t m_chunk_start; uint64_t m_chunk_end; @@ -1647,11 +1654,11 @@ struct read_data { public: input(const std::string& path, int64_t offset, uint64_t host_id, - uint64_t host_size, uint64_t chunk_n, uint64_t chunk_start, - uint64_t chunk_end, uint64_t total_chunk_size, - const hermes::exposed_memory& buffers) + uint64_t host_size, const std::string& wbitset, uint64_t chunk_n, + uint64_t chunk_start, uint64_t chunk_end, + uint64_t total_chunk_size, const hermes::exposed_memory& buffers) : m_path(path), m_offset(offset), m_host_id(host_id), - m_host_size(host_size), m_chunk_n(chunk_n), + m_host_size(host_size), m_wbitset(wbitset), m_chunk_n(chunk_n), m_chunk_start(chunk_start), m_chunk_end(chunk_end), m_total_chunk_size(total_chunk_size), m_buffers(buffers) {} @@ -1685,6 +1692,11 @@ struct read_data { return m_host_size; } + std::string + wbitset() const { + return m_wbitset; + } + uint64_t chunk_n() const { return m_chunk_n; @@ -1713,15 +1725,16 @@ struct read_data { explicit input(const rpc_read_data_in_t& other) : m_path(other.path), m_offset(other.offset), m_host_id(other.host_id), m_host_size(other.host_size), - m_chunk_n(other.chunk_n), m_chunk_start(other.chunk_start), - m_chunk_end(other.chunk_end), + m_wbitset(other.wbitset), m_chunk_n(other.chunk_n), + m_chunk_start(other.chunk_start), m_chunk_end(other.chunk_end), m_total_chunk_size(other.total_chunk_size), m_buffers(other.bulk_handle) {} explicit operator rpc_read_data_in_t() { - return {m_path.c_str(), m_offset, m_host_id, - m_host_size, m_chunk_n, m_chunk_start, - m_chunk_end, m_total_chunk_size, hg_bulk_t(m_buffers)}; + return {m_path.c_str(), m_offset, m_host_id, + m_host_size, m_wbitset.c_str(), m_chunk_n, + m_chunk_start, m_chunk_end, m_total_chunk_size, + hg_bulk_t(m_buffers)}; } private: @@ -1729,6 +1742,7 @@ struct read_data { int64_t m_offset; uint64_t m_host_id; uint64_t m_host_size; + std::string m_wbitset; uint64_t m_chunk_n; uint64_t m_chunk_start; uint64_t m_chunk_end; diff --git a/include/common/metadata.hpp b/include/common/metadata.hpp index d64f94921439be0d738c6dda01b7059f94457dbf..25ed22f71428323099f461b85315a5357d12ab60 100644 --- a/include/common/metadata.hpp +++ b/include/common/metadata.hpp @@ -37,7 +37,6 @@ #include #include - namespace gkfs::metadata { constexpr mode_t LINK_MODE = ((S_IRWXU | S_IRWXG | S_IRWXO) | S_IFLNK); diff --git a/include/common/rpc/distributor.hpp b/include/common/rpc/distributor.hpp index b42d1ae30b90697e8fe449562525695f27d55531..697a1df6c5ebf21756081dbd0851f083e5466fed 100644 --- a/include/common/rpc/distributor.hpp +++ b/include/common/rpc/distributor.hpp @@ -36,7 +36,6 @@ #include #include #include - namespace gkfs::rpc { using chunkid_t = unsigned int; @@ -48,15 +47,20 @@ public: localhost() const = 0; virtual host_t - locate_data(const std::string& path, const chunkid_t& chnk_id) const = 0; + locate_data(const std::string& path, const chunkid_t& chnk_id, + const int num_copy) const = 0; // TODO: We need to pass hosts_size in the server side, because the number // of servers are not defined (in startup) + + virtual unsigned int + hosts_size() const = 0; + virtual host_t locate_data(const std::string& path, const chunkid_t& chnk_id, - unsigned int hosts_size) = 0; + unsigned int hosts_size, const int num_copy) = 0; virtual host_t - locate_file_metadata(const std::string& path) const = 0; + locate_file_metadata(const std::string& path, const int num_copy) const = 0; virtual std::vector locate_directory_metadata(const std::string& path) const = 0; @@ -75,19 +79,59 @@ public: SimpleHashDistributor(host_t localhost, unsigned int hosts_size); + unsigned int + hosts_size() const override; + host_t localhost() const override; host_t - locate_data(const std::string& path, - const chunkid_t& chnk_id) const override; + locate_data(const std::string& path, const chunkid_t& chnk_id, + const int num_copy) const override; host_t locate_data(const std::string& path, const chunkid_t& chnk_id, - unsigned int host_size); + unsigned int host_size, const int num_copy); host_t - locate_file_metadata(const std::string& path) const override; + locate_file_metadata(const std::string& path, + const int num_copy) const override; + + std::vector + locate_directory_metadata(const std::string& path) const override; +}; + +class ECCDistributor : public Distributor { +private: + host_t localhost_; + unsigned int hosts_size_{0}; + unsigned int ecc_size_{0}; + std::vector all_hosts_; + std::hash str_hash; + +public: + ECCDistributor(); + + ECCDistributor(host_t localhost, unsigned int hosts_size, + unsigned int ecc_size); + + unsigned int + hosts_size() const override; + + host_t + localhost() const override; + + host_t + locate_data(const std::string& path, const chunkid_t& chnk_id, + const int num_copy) const override; + + host_t + locate_data(const std::string& path, const chunkid_t& chnk_id, + unsigned int host_size, const int num_copy); + + host_t + locate_file_metadata(const std::string& path, + const int num_copy) const override; std::vector locate_directory_metadata(const std::string& path) const override; @@ -96,6 +140,7 @@ public: class LocalOnlyDistributor : public Distributor { private: host_t localhost_; + unsigned int hosts_size_{0}; public: explicit LocalOnlyDistributor(host_t localhost); @@ -103,12 +148,16 @@ public: host_t localhost() const override; + unsigned int + hosts_size() const override; + host_t - locate_data(const std::string& path, - const chunkid_t& chnk_id) const override; + locate_data(const std::string& path, const chunkid_t& chnk_id, + const int num_copy) const override; host_t - locate_file_metadata(const std::string& path) const override; + locate_file_metadata(const std::string& path, + const int num_copy) const override; std::vector locate_directory_metadata(const std::string& path) const override; @@ -117,7 +166,7 @@ public: class ForwarderDistributor : public Distributor { private: host_t fwd_host_; - unsigned int hosts_size_; + unsigned int hosts_size_{0}; std::vector all_hosts_; std::hash str_hash; @@ -127,16 +176,20 @@ public: host_t localhost() const override final; + unsigned int + hosts_size() const override; + host_t - locate_data(const std::string& path, - const chunkid_t& chnk_id) const override final; + locate_data(const std::string& path, const chunkid_t& chnk_id, + const int num_copy) const override final; host_t locate_data(const std::string& path, const chunkid_t& chnk_id, - unsigned int host_size) override final; + unsigned int host_size, const int num_copy) override final; host_t - locate_file_metadata(const std::string& path) const override; + locate_file_metadata(const std::string& path, + const int num_copy) const override; std::vector locate_directory_metadata(const std::string& path) const override; @@ -176,16 +229,20 @@ public: host_t localhost() const override; + unsigned int + hosts_size() const override; + host_t - locate_data(const std::string& path, - const chunkid_t& chnk_id) const override; + locate_data(const std::string& path, const chunkid_t& chnk_id, + const int num_copy) const override; host_t locate_data(const std::string& path, const chunkid_t& chnk_id, - unsigned int host_size); + unsigned int host_size, const int num_copy); host_t - locate_file_metadata(const std::string& path) const override; + locate_file_metadata(const std::string& path, + const int num_copy) const override; std::vector locate_directory_metadata(const std::string& path) const override; diff --git a/include/common/rpc/rpc_types.hpp b/include/common/rpc/rpc_types.hpp index 19612c45c0fdfe0f71c8bfecca45bb368da48a78..d28798d52795a2717c0ae2911020fbfc3e2c4c81 100644 --- a/include/common/rpc/rpc_types.hpp +++ b/include/common/rpc/rpc_types.hpp @@ -89,9 +89,9 @@ MERCURY_GEN_PROC( rpc_read_data_in_t, ((hg_const_string_t) (path))((int64_t) (offset))( (hg_uint64_t) (host_id))((hg_uint64_t) (host_size))( - (hg_uint64_t) (chunk_n))((hg_uint64_t) (chunk_start))( - (hg_uint64_t) (chunk_end))((hg_uint64_t) (total_chunk_size))( - (hg_bulk_t) (bulk_handle))) + (hg_const_string_t) (wbitset))((hg_uint64_t) (chunk_n))( + (hg_uint64_t) (chunk_start))((hg_uint64_t) (chunk_end))( + (hg_uint64_t) (total_chunk_size))((hg_bulk_t) (bulk_handle))) MERCURY_GEN_PROC(rpc_data_out_t, ((int32_t) (err))((hg_size_t) (io_size))) @@ -99,9 +99,9 @@ MERCURY_GEN_PROC( rpc_write_data_in_t, ((hg_const_string_t) (path))((int64_t) (offset))( (hg_uint64_t) (host_id))((hg_uint64_t) (host_size))( - (hg_uint64_t) (chunk_n))((hg_uint64_t) (chunk_start))( - (hg_uint64_t) (chunk_end))((hg_uint64_t) (total_chunk_size))( - (hg_bulk_t) (bulk_handle))) + (hg_const_string_t) (wbitset))((hg_uint64_t) (chunk_n))( + (hg_uint64_t) (chunk_start))((hg_uint64_t) (chunk_end))( + (hg_uint64_t) (total_chunk_size))((hg_bulk_t) (bulk_handle))) MERCURY_GEN_PROC(rpc_get_dirents_in_t, ((hg_const_string_t) (path))((hg_bulk_t) (bulk_handle))) diff --git a/include/common/rpc/rpc_util.hpp b/include/common/rpc/rpc_util.hpp index eb595596c4974b6e0aa6810794c056f52cd7b8fc..53e9aafb3adc4b8a63e91417ebb404edce52a142 100644 --- a/include/common/rpc/rpc_util.hpp +++ b/include/common/rpc/rpc_util.hpp @@ -35,6 +35,9 @@ extern "C" { } #include +#include +#include +#include namespace gkfs::rpc { @@ -49,6 +52,18 @@ std::string get_host_by_name(const std::string& hostname); #endif +bool +get_bitset(const std::vector& data, const uint16_t position); + +void +set_bitset(std::vector& data, const uint16_t position); + +std::string +compressBitset(const std::vector& bytes); + +std::vector +decompressBitset(const std::string& compressedString); + } // namespace gkfs::rpc #endif // GEKKOFS_COMMON_RPC_UTILS_HPP diff --git a/scripts/profiles/0.9.3-exp/agios.specs b/scripts/profiles/0.9.3-exp/agios.specs new file mode 100644 index 0000000000000000000000000000000000000000..e967029cd6b56035f8a9fbfeb4f19b401c29ab37 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/agios.specs @@ -0,0 +1,81 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies (except transport-specific and experimental)" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3-exp/all.specs b/scripts/profiles/0.9.3-exp/all.specs new file mode 100644 index 0000000000000000000000000000000000000000..e6ad862db076166a51893331c8e2c2d58f652cfe --- /dev/null +++ b/scripts/profiles/0.9.3-exp/all.specs @@ -0,0 +1,89 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" + ["curl"]="7.82.0" + ["prometheus-cpp"]="v1.0.0" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" + ["gf-complete"]="a6862d10c9db467148f20eef2c6445ac9afd94d8" + ["Jerasure"]="414c96ef2b9934953b6facb31d803d79b1dd1405" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "agios" "curl" "prometheus-cpp" "parallax" "gf-complete" "Jerasure" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3-exp/ci.specs b/scripts/profiles/0.9.3-exp/ci.specs new file mode 100644 index 0000000000000000000000000000000000000000..1266c4f33ad1ffe2842bdfd5110730b846f502e2 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/ci.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies required by the CI" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["prometheus-cpp"]="v1.0.0" + ) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" + ["gf-complete"]="a6862d10c9db467148f20eef2c6445ac9afd94d8" + ["Jerasure"]="414c96ef2b9934953b6facb31d803d79b1dd1405" + ) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "date" "agios" "parallax" "prometheus-cpp" "gf-complete" "Jerasure" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3-exp/default.specs b/scripts/profiles/0.9.3-exp/default.specs new file mode 100644 index 0000000000000000000000000000000000000000..afc567273e8c5d77fc3e73e604677a0c8a0ebe4a --- /dev/null +++ b/scripts/profiles/0.9.3-exp/default.specs @@ -0,0 +1,80 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch syscall_intercept_clone3.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3-exp/direct.specs b/scripts/profiles/0.9.3-exp/direct.specs new file mode 100644 index 0000000000000000000000000000000000000000..6f2a48123e3ae1f3ec0d9e55ee9478c16dc5daf2 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/direct.specs @@ -0,0 +1,76 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Direct dependencies" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3-exp/infiniband_verbs.specs b/scripts/profiles/0.9.3-exp/infiniband_verbs.specs new file mode 100644 index 0000000000000000000000000000000000000000..c34c4ff6d8c76566843fcb0bb4471dc0d0e3cb1a --- /dev/null +++ b/scripts/profiles/0.9.3-exp/infiniband_verbs.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Infiniband supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric%verbs"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading/installing +order=( + "lz4" "capstone" "json-c" "libfabric%verbs" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric%verbs"]="--enable-verbs=yes" +) diff --git a/scripts/profiles/0.9.3-exp/install/Jerasure.install b/scripts/profiles/0.9.3-exp/install/Jerasure.install new file mode 100644 index 0000000000000000000000000000000000000000..5eda7d3f3774b6dbd0d3520792a45d981d3ad2e2 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/Jerasure.install @@ -0,0 +1,59 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="Jerasure" + CURR="${SOURCE_DIR}/${ID}" + cd "${CURR}" + autoreconf -fi + ./configure --prefix="${INSTALL_DIR}" + make -j"${CORES}" + make install + mv ${INSTALL_DIR}/include/jerasure.h ${INSTALL_DIR}/include/jerasure/jerasure.h +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3-exp/install/agios.install b/scripts/profiles/0.9.3-exp/install/agios.install new file mode 100644 index 0000000000000000000000000000000000000000..3d0392ff5305710794c19dc1e8b96985722fa5b7 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/agios.install @@ -0,0 +1,57 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="agios" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" .. + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3-exp/install/argobots.install b/scripts/profiles/0.9.3-exp/install/argobots.install new file mode 100644 index 0000000000000000000000000000000000000000..96ccea920eeb4493c7e6287737cf091945e6b06a --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/argobots.install @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="argobots" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" --enable-perf-opt --disable-checks + make -j"${CORES}" + make install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3-exp/install/capstone.install b/scripts/profiles/0.9.3-exp/install/capstone.install new file mode 100644 index 0000000000000000000000000000000000000000..ff299d82b47f76389b3650553014438c65e92ae7 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/capstone.install @@ -0,0 +1,57 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="capstone" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3-exp/install/curl.install b/scripts/profiles/0.9.3-exp/install/curl.install new file mode 100644 index 0000000000000000000000000000000000000000..d873819c4521e1ed8dce128c2699cc764356f92a --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/curl.install @@ -0,0 +1,58 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="curl" + CURR="${SOURCE_DIR}/${ID}" + cd "${CURR}" + autoreconf -fi + ./configure --prefix="${INSTALL_DIR}" --without-ssl + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3-exp/install/date.install b/scripts/profiles/0.9.3-exp/install/date.install new file mode 100644 index 0000000000000000000000000000000000000000..c4f12524cf738888185be97b2c2a7c3aed4f6a04 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/date.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="date" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_CXX_STANDARD:STRING=14 \ + -DUSE_SYSTEM_TZ_DB:BOOL=ON \ + -DBUILD_SHARED_LIBS:BOOL=ON .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3-exp/install/gf-complete.install b/scripts/profiles/0.9.3-exp/install/gf-complete.install new file mode 100644 index 0000000000000000000000000000000000000000..70d675a2c28337bb5f10c928ec857694fc9d45d1 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/gf-complete.install @@ -0,0 +1,58 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="gf-complete" + CURR="${SOURCE_DIR}/${ID}" + cd "${CURR}" + autoreconf -fi + ./configure --prefix="${INSTALL_DIR}" + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3-exp/install/json-c.install b/scripts/profiles/0.9.3-exp/install/json-c.install new file mode 100644 index 0000000000000000000000000000000000000000..7280cda81e54d91500ef5137a45ad56cad298bc1 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/json-c.install @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/json-c" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + cmake -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install + # Margo doesn't search in both directories, so we make it available in both lib and lib64 + if [[ -f "${INSTALL_DIR}/lib64/pkgconfig/json-c.pc" ]]; then + cp ${INSTALL_DIR}/lib64/pkgconfig/json-c.pc ${INSTALL_DIR}/lib/pkgconfig/ + fi +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3-exp/install/libfabric%verbs.install b/scripts/profiles/0.9.3-exp/install/libfabric%verbs.install new file mode 100644 index 0000000000000000000000000000000000000000..15257eb817549eefcab8264671c5df97307e49db --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/libfabric%verbs.install @@ -0,0 +1,67 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="libfabric%verbs" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3-exp/install/libfabric.install b/scripts/profiles/0.9.3-exp/install/libfabric.install new file mode 100644 index 0000000000000000000000000000000000000000..7db89438e194931bb6311ed581e8d0d81576fa85 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/libfabric.install @@ -0,0 +1,67 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID=libfabric + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3-exp/install/lz4.install b/scripts/profiles/0.9.3-exp/install/lz4.install new file mode 100644 index 0000000000000000000000000000000000000000..13e4e53b381f7df45aac175f3a9af84498b8abd4 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/lz4.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="lz4" + CURR="${SOURCE_DIR}/${ID}" + cd "${CURR}" + # try to remove binaries first in case they already exist. Otherwise install fails. + LZ4_BINS=("${INSTALL_DIR}"/bin/lz4c "${INSTALL_DIR}"/bin/lz4cat "${INSTALL_DIR}"/bin/unlz4 "${INSTALL_DIR}"/bin/lz4) + for LZ4_BIN in "${LZ4_BINS[@]}"; do + [ -e "$LZ4_BIN" ] && rm "$LZ4_BIN" + done + make -j"${CORES}" + make DESTDIR="${INSTALL_DIR}" PREFIX="" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3-exp/install/margo.install b/scripts/profiles/0.9.3-exp/install/margo.install new file mode 100644 index 0000000000000000000000000000000000000000..d2c6e4b5f2d21ff0e0f02865c53fe374d161b976 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/margo.install @@ -0,0 +1,59 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="margo" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./prepare.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" CFLAGS="${CFLAGS} -Wall -O3" + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3-exp/install/mercury.install b/scripts/profiles/0.9.3-exp/install/mercury.install new file mode 100644 index 0000000000000000000000000000000000000000..601305b830c8cf1e320ca81cbdb45368b335cc3c --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/mercury.install @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + # if the profile compiles bmi, enable it + if [[ -n "${PROFILE_DEP_NAMES['bmi']}" ]]; then + USE_BMI="-DNA_USE_BMI:BOOL=ON" + else + USE_BMI="-DNA_USE_BMI:BOOL=OFF" + fi + + # if the profile provides any flavour of libfabric, enable it + if profile_has_dependency "^libfabric.*$"; then + USE_OFI="-DNA_USE_OFI:BOOL=ON" + else + USE_OFI="-DNA_USE_OFI:BOOL=OFF" + fi + + ID="mercury" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_PREFIX_PATH=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DBUILD_TESTING:BOOL=ON \ + -DMERCURY_USE_CHECKSUMS:BOOL=OFF \ + -DMERCURY_USE_BOOST_PP:BOOL=ON \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + ${USE_BMI} ${USE_OFI} \ + .. + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3-exp/install/parallax.install b/scripts/profiles/0.9.3-exp/install/parallax.install new file mode 100644 index 0000000000000000000000000000000000000000..38bd9c68961003c68762054edecd89586bcf2bda --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/parallax.install @@ -0,0 +1,64 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + CURR="${SOURCE_DIR}/parallax" + # sed -i -e 's/KEY_SIZE (256)/KEY_SIZE (4096)/g' ${CURR}/lib/btree/conf.h + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_CXX_FLAGS_RELEASE="-Wno-error=unused-result" \ + -DDISABLE_LOGGING:BOOL=ON \ + .. + make -j"${CORES}" + make install + # We need to copy this file as it is not installed + cp ${CURR}/lib/include/parallax/structures.h ${INSTALL_DIR}/include/ +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3-exp/install/prometheus-cpp.install b/scripts/profiles/0.9.3-exp/install/prometheus-cpp.install new file mode 100644 index 0000000000000000000000000000000000000000..62e420217339c0217176db6aecec3a549417df6c --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/prometheus-cpp.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="prometheus-cpp" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3-exp/install/rocksdb.install b/scripts/profiles/0.9.3-exp/install/rocksdb.install new file mode 100644 index 0000000000000000000000000000000000000000..6ece050bd753ff3474c69ac1e013cd541e4d8ff7 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/rocksdb.install @@ -0,0 +1,79 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CXXFLAGS='' + # gcc 9 and clang 8 need -Wno-error=deprecated-copy -Wno-error=pessimizing-move + if [[ ("${COMPILER_NAME}" == "g++" && "${COMPILER_MAJOR_VERSION}" -ge 9) || + ("${COMPILER_NAME}" == "clang" && "${COMPILER_MAJOR_VERSION}" -ge 8) ]]; then + CXXFLAGS='-Wno-error=deprecated-copy -Wno-error=pessimizing-move -Wno-error=maybe-uninitialized' + fi + + # TODO use SSE? + CURR="${SOURCE_DIR}/rocksdb" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR="${INSTALL_DIR}/lib" \ + -DCMAKE_INSTALL_INCLUDEDIR="${INSTALL_DIR}/include" \ + -DROCKSDB_BUILD_SHARED=OFF \ + -DWITH_LZ4=ON \ + -DWITH_GFLAGS=OFF \ + -DUSE_RTTI=1 \ + -DPORTABLE=1 \ + -DWITH_ALL_TESTS=OFF \ + -DWITH_BENCHMARK_TOOLS=OFF \ + -DWITH_TOOLS=OFF .. + make -j"${CORES}" install + + +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3-exp/install/syscall_intercept.install b/scripts/profiles/0.9.3-exp/install/syscall_intercept.install new file mode 100644 index 0000000000000000000000000000000000000000..9dd287271c61838a29cedd5e3bf3146ffb598f08 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/install/syscall_intercept.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="syscall_intercept" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}"/build + $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + -DBUILD_EXAMPLES:BOOL=OFF \ + -DBUILD_TESTS:BOOK=OFF .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3-exp/marenostrum4.specs b/scripts/profiles/0.9.3-exp/marenostrum4.specs new file mode 100644 index 0000000000000000000000000000000000000000..2fd272eff2539038687724bb33bde698e9241277 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/marenostrum4.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Marenostrum 4 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" +) diff --git a/scripts/profiles/0.9.3-exp/mogon2.specs b/scripts/profiles/0.9.3-exp/mogon2.specs new file mode 100644 index 0000000000000000000000000000000000000000..71ebf7a65e326a93b03fce9f283d4e80d7fc0d36 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/mogon2.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3-exp/ngio.specs b/scripts/profiles/0.9.3-exp/ngio.specs new file mode 100644 index 0000000000000000000000000000000000000000..72c99cebdb4f79b33b86c81f618f869a5ff674d2 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/ngio.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for NEXTGenIO prototype cluster" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3-exp/omnipath_psm2.specs b/scripts/profiles/0.9.3-exp/omnipath_psm2.specs new file mode 100644 index 0000000000000000000000000000000000000000..10163c302156f7e50faac52a8337068c615461e8 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/omnipath_psm2.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Omnipath supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3-exp/p9.specs b/scripts/profiles/0.9.3-exp/p9.specs new file mode 100644 index 0000000000000000000000000000000000000000..6c78aa1ee8e20a8cbbc23b79b1fbbb3ea5113318 --- /dev/null +++ b/scripts/profiles/0.9.3-exp/p9.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for PowerPC supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="6.26.1" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" +) diff --git a/scripts/profiles/sources.list b/scripts/profiles/sources.list index 3c10823725390c576564ac18b9812b0b57201359..fcc2f4ed0ac7ad8f2b7634379963b3c314720f28 100644 --- a/scripts/profiles/sources.list +++ b/scripts/profiles/sources.list @@ -53,6 +53,8 @@ sources=( ["parallax"]="https://github.com/CARV-ICS-FORTH/parallax.git" ["prometheus-cpp"]="https://github.com/jupp0r/prometheus-cpp/releases/download/{{VERSION}}/prometheus-cpp-with-submodules.tar.gz" ["curl"]="https://curl.se/download/curl-{{VERSION}}.tar.gz" + ["gf-complete"]="https://github.com/ceph/gf-complete" + ["Jerasure"]="https://github.com/tsuraan/Jerasure" ) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 536d91165b5ac251626c14ae2cf4d23f3f5ac295..8c3f117e9267a1dafafa34c037bd312a3a21d8e0 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -64,6 +64,22 @@ target_link_libraries( Date::TZ ) +if(GKFS_ENABLE_EC) + target_link_libraries( + gkfs_intercept + PRIVATE metadata distributor env_util arithmetic path_util rpc_utils + PUBLIC Syscall_intercept::Syscall_intercept + dl + Mercury::Mercury + hermes + fmt::fmt + Threads::Threads + Date::TZ + GF_complete::GF_complete + Jerasure::Jerasure + ) +endif() + install( TARGETS gkfs_intercept LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 282a3eae95ce173ae6add0f735a704ba61797393..d5120ab13e4f1ba562808ac651cb290aa5c7f9df 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -45,6 +45,11 @@ extern "C" { #include } +#ifdef GKFS_ENABLE_EC +#include +#include +#endif + using namespace std; /* @@ -298,11 +303,21 @@ gkfs_create(const std::string& path, mode_t mode) { if(check_parent_dir(path)) { return -1; } - auto err = gkfs::rpc::forward_create(path, mode); - if(err) { - errno = err; + // Write to all replicas, at least one need to success + bool success = false; + for(auto copy = 0; copy < CTX->get_replicas() + 1; copy++) { + auto err = gkfs::rpc::forward_create(path, mode, copy); + if(err) { + errno = err; + } else { + success = true; + errno = 0; + } + } + if(!success) { return -1; } + return 0; } @@ -340,7 +355,7 @@ gkfs_remove(const std::string& path) { return -1; } } - auto err = gkfs::rpc::forward_remove(new_path); + auto err = gkfs::rpc::forward_remove(new_path, CTX->get_replicas()); if(err) { errno = err; return -1; @@ -350,7 +365,7 @@ gkfs_remove(const std::string& path) { #endif // HAS_RENAME #endif // HAS_SYMLINKS - auto err = gkfs::rpc::forward_remove(path); + auto err = gkfs::rpc::forward_remove(path, CTX->get_replicas()); if(err) { errno = err; return -1; @@ -406,6 +421,7 @@ gkfs_access(const std::string& path, const int mask, bool follow_links) { * We use blocks to determine if the file is a renamed file. * If the file is re-renamed (a->b->a) a recovers the block of b * and we delete b. + * There is no support for replication in rename * @param old_path * @param new_path * @return 0 on success, -1 on failure @@ -441,14 +457,14 @@ gkfs_rename(const string& old_path, const string& new_path) { md_old.value().target_path(""); auto err = gkfs::rpc::forward_update_metadentry( - new_path, md_old.value(), flags); + new_path, md_old.value(), flags, 0); if(err) { errno = err; return -1; } // Delete old file - err = gkfs::rpc::forward_remove(old_path); + err = gkfs::rpc::forward_remove(old_path, CTX->get_replicas()); if(err) { errno = err; return -1; @@ -674,7 +690,9 @@ gkfs_lseek(shared_ptr gkfs_fd, off_t offset, gkfs_fd->pos(gkfs_fd->pos() + offset); break; case SEEK_END: { - auto ret = gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path()); + // TODO: handle replicas + auto ret = + gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path(), 0); auto err = ret.first; if(err) { errno = err; @@ -723,14 +741,17 @@ gkfs_truncate(const std::string& path, off_t old_size, off_t new_size) { if(new_size == old_size) { return 0; } - auto err = gkfs::rpc::forward_decr_size(path, new_size); - if(err) { - LOG(DEBUG, "Failed to decrease size"); - errno = err; - return -1; + for(auto copy = 0; copy < (CTX->get_replicas() + 1); copy++) { + auto err = gkfs::rpc::forward_decr_size(path, new_size, copy); + if(err) { + LOG(DEBUG, "Failed to decrease size"); + errno = err; + return -1; + } } - err = gkfs::rpc::forward_truncate(path, old_size, new_size); + auto err = gkfs::rpc::forward_truncate(path, old_size, new_size, + CTX->get_replicas()); if(err) { LOG(DEBUG, "Failed to truncate data"); errno = err; @@ -842,6 +863,116 @@ gkfs_dup2(const int oldfd, const int newfd) { return CTX->file_map()->dup2(oldfd, newfd); } +#ifdef GKFS_ENABLE_EC +/** + * @brief Compute and store the erasure codes of a chunk line, smaller files are + ignored. The ec are stored in [path]ecc_chunkstart_chunkend files + * + * @param file + * @param count + * @param offset + * @param updated_size + * @return true if the computation was successful + * @return false if the computation was not successful + * + * For each chunk we will have a set of chunks involved on that calculation + [0] [1] [2] [3] [4] [n-p] [p1] [p2] + [n-p+1] .... + i.e. : [0] -> 1,2,3,4,n-p + i.e : [4] -> 0,1,2,3,n-p + i.e : [n-p+1] +*/ +bool +gkfs_ecc_write(std::shared_ptr file, size_t count, + off64_t offset, off64_t updated_size) { + auto path = make_shared(file->path()); + auto append_flag = file->get_flag(gkfs::filemap::OpenFile_flags::append); + auto chunks = gkfs::rpc::calc_op_chunks(*path, append_flag, offset, count, + updated_size); + + std::set chunk_set; + + + if((uint64_t) updated_size >= + (uint64_t) CTX->hosts().size() * gkfs::config::rpc::chunksize) { + auto data_servers = CTX->hosts().size() - CTX->get_replicas(); + for(auto i = chunks.first; i <= chunks.second; ++i) { + auto initial_row_chunk = (i / data_servers) * data_servers; + chunk_set.insert(initial_row_chunk); + } + + // 1 - Read data from the other chunks + + char** data = (char**) malloc(sizeof(char*) * data_servers); + char** coding = (char**) malloc(sizeof(char*) * CTX->get_replicas()); + + for(unsigned int i = 0; i < data_servers; ++i) { + data[i] = (char*) malloc(gkfs::config::rpc::chunksize); + } + for(auto i = 0; i < CTX->get_replicas(); ++i) { + coding[i] = (char*) malloc(gkfs::config::rpc::chunksize); + } + + LOG(DEBUG, + "Operation Size {} - Range {}-{} - data_servers {} replica_servers {}", + updated_size, chunks.first, chunks.second, data_servers, + CTX->get_replicas()); + + // TODO : This could be optimised, with a single read loop + for(auto i : chunk_set) { + + for(auto j = i; j < i + data_servers; ++j) { + std::set failed; + LOG(DEBUG, "Reading Chunk {} -> {}", i, j); + + auto out = gkfs::rpc::forward_read( + *path, data[j - i], j * gkfs::config::rpc::chunksize, + gkfs::config::rpc::chunksize, 0, failed); + if(out.first != 0) { + LOG(ERROR, "Read Parity Error: {}", out.first); + } + } + + // We have all the data to process a EC + + int* matrix = reed_sol_vandermonde_coding_matrix( + data_servers, CTX->get_replicas(), 8); + jerasure_matrix_encode(data_servers, CTX->get_replicas(), 8, matrix, + data, coding, gkfs::config::rpc::chunksize); + + LOG(DEBUG, "EC computation finished"); + free(matrix); + // Write erasure + std::string ecc_path = file->path() + "_ecc_" + to_string(i) + "_" + + to_string(i + data_servers - 1); + for(int i = 0; i < CTX->get_replicas(); i++) { + auto ecc_write = gkfs::rpc::ecc_forward_write( + ecc_path, coding[i], gkfs::config::rpc::chunksize, + i + data_servers); + if(ecc_write.first != 0) { + LOG(ERROR, "write Parity Error: {}", ecc_write.first); + return false; + } else { + LOG(DEBUG, "write Parity OK: {}", ecc_path); + } + } + } + for(unsigned int i = 0; i < data_servers; ++i) { + free(data[i]); + } + for(auto i = 0; i < CTX->get_replicas(); ++i) { + free(coding[i]); + } + + free(coding); + free(data); + } else { + LOG(DEBUG, "No EC in small files"); + return false; + } + return true; +} +#endif /** * Wrapper function for all gkfs write operations * errno may be set @@ -864,9 +995,11 @@ gkfs_pwrite(std::shared_ptr file, const char* buf, } auto path = make_unique(file->path()); auto is_append = file->get_flag(gkfs::filemap::OpenFile_flags::append); + auto write_size = 0; + auto num_replicas = CTX->get_replicas(); auto ret_offset = gkfs::rpc::forward_update_metadentry_size( - *path, count, offset, is_append); + *path, count, offset, is_append, num_replicas); auto err = ret_offset.first; if(err) { LOG(ERROR, "update_metadentry_size() failed with err '{}'", err); @@ -888,8 +1021,28 @@ gkfs_pwrite(std::shared_ptr file, const char* buf, offset = ret_offset.second; } - auto ret_write = gkfs::rpc::forward_write(*path, buf, offset, count); + auto ret_write = gkfs::rpc::forward_write(*path, buf, offset, count, 0); err = ret_write.first; + write_size = ret_write.second; + +#ifdef GKFS_ENABLE_EC + // Only compute Erasure codes if we do not have enabled the ondemand + // environment variable + if(CTX->get_ec_ondemand() == false) { + auto res = gkfs_ecc_write(file, count, offset, write_size); + if(res) { + LOG(ERROR, "erasure code writing failed"); + } + } +#else + if(num_replicas > 0) { + + auto ret_write_repl = gkfs::rpc::forward_write(*path, buf, offset, + count, num_replicas); + write_size = ret_write_repl.second; + } +} +#endif if(err) { LOG(WARNING, "gkfs::rpc::forward_write() failed with err '{}'", err); errno = err; @@ -897,14 +1050,14 @@ gkfs_pwrite(std::shared_ptr file, const char* buf, } if(update_pos) { // Update offset in file descriptor in the file map - file->pos(offset + ret_write.second); + file->pos(offset + write_size); } - if(static_cast(ret_write.second) != count) { + if(static_cast(write_size) != count) { LOG(WARNING, "gkfs::rpc::forward_write() wrote '{}' bytes instead of '{}'", - ret_write.second, count); + write_size, count); } - return ret_write.second; // return written size + return write_size; // return written size } /** @@ -1019,12 +1172,33 @@ gkfs_pread(std::shared_ptr file, char* buf, return -1; } - // Zeroing buffer before read is only relevant for sparse files. Otherwise - // sparse regions contain invalid data. + // Zeroing buffer before read is only relevant for sparse files. + // Otherwise sparse regions contain invalid data. if constexpr(gkfs::config::io::zero_buffer_before_read) { memset(buf, 0, sizeof(char) * count); } - auto ret = gkfs::rpc::forward_read(file->path(), buf, offset, count); + std::pair ret; + std::set failed; // set with failed targets. + if(CTX->get_replicas() != 0) { + + ret = gkfs::rpc::forward_read(file->path(), buf, offset, count, + CTX->get_replicas(), failed); + while(ret.first == EIO) { +#ifdef GKFS_ENABLE_EC + LOG(WARNING, "failed to read"); +#else + ret = gkfs::rpc::forward_read(file->path(), buf, offset, count, + CTX->get_replicas(), failed); + LOG(WARNING, "gkfs::rpc::forward_read() failed with ret '{}'", + ret.first); +#endif + } + + } else { + ret = gkfs::rpc::forward_read(file->path(), buf, offset, count, 0, + failed); + } + auto err = ret.first; if(err) { LOG(WARNING, "gkfs::rpc::forward_read() failed with ret '{}'", err); @@ -1192,7 +1366,7 @@ gkfs_rmdir(const std::string& path) { errno = ENOTEMPTY; return -1; } - err = gkfs::rpc::forward_remove(path); + err = gkfs::rpc::forward_remove(path, CTX->get_replicas()); if(err) { errno = err; return -1; @@ -1234,11 +1408,11 @@ gkfs_getdents(unsigned int fd, struct linux_dirent* dirp, unsigned int count) { * Calculate the total dentry size within the kernel struct * `linux_dirent` depending on the file name size. The size is then * aligned to the size of `long` boundary. This line was originally - * defined in the linux kernel: fs/readdir.c in function filldir(): int - * reclen = ALIGN(offsetof(struct linux_dirent, d_name) + namlen + 2, - * sizeof(long)); However, since d_name is null-terminated and - * de.name().size() does not include space for the null-terminator, we - * add 1. Thus, + 3 in total. + * defined in the linux kernel: fs/readdir.c in function filldir(): + * int reclen = ALIGN(offsetof(struct linux_dirent, d_name) + namlen + * + 2, sizeof(long)); However, since d_name is null-terminated and + * de.name().size() does not include space for the null-terminator, + * we add 1. Thus, + 3 in total. */ auto total_size = ALIGN(offsetof(struct linux_dirent, d_name) + de.name().size() + 3, @@ -1305,13 +1479,14 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, * `linux_dirent` depending on the file name size. The size is then * aligned to the size of `long` boundary. * - * This line was originally defined in the linux kernel: fs/readdir.c in - * function filldir64(): int reclen = ALIGN(offsetof(struct - * linux_dirent64, d_name) + namlen + 1, sizeof(u64)); We keep + 1 - * because: Since d_name is null-terminated and de.name().size() does - * not include space for the null-terminator, we add 1. Since d_name in - * our `struct linux_dirent64` definition is not a zero-size array (as - * opposed to the kernel version), we subtract 1. Thus, it stays + 1. + * This line was originally defined in the linux kernel: + * fs/readdir.c in function filldir64(): int reclen = + * ALIGN(offsetof(struct linux_dirent64, d_name) + namlen + 1, + * sizeof(u64)); We keep + 1 because: Since d_name is + * null-terminated and de.name().size() does not include space for + * the null-terminator, we add 1. Since d_name in our `struct + * linux_dirent64` definition is not a zero-size array (as opposed + * to the kernel version), we subtract 1. Thus, it stays + 1. */ auto total_size = ALIGN(offsetof(struct linux_dirent64, d_name) + de.name().size() + 1, @@ -1489,3 +1664,29 @@ gkfs_getsingleserverdir(const char* path, struct dirent_extended* dirp, } return written; } + +#ifdef GKFS_ENABLE_EC +/** + * This function defines an extension to calculate the erasure codes of a file + * Returns 0 on success, -1 on failure + */ +extern "C" int +gkfs_ec_ondemand(const unsigned int fd) { + if(CTX->file_map()->exist(fd)) { + auto path = CTX->file_map()->get(fd)->path(); + auto file = CTX->file_map()->get(fd); + auto md = gkfs::utils::get_metadata(path); + if(!md) { + return -1; + } + auto count = md->size(); + auto res = gkfs::syscall::gkfs_ecc_write(file, count, 0, count); + if(!res) { + return -1; + } + + return 0; + } + return -1; +} +#endif \ No newline at end of file diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 7b05ca9a3fa416b66a97d2985f2e6c0856c61049..25056a86d34643dadaf134f11f1095d611ebbfa6 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -223,6 +223,9 @@ init_environment() { #ifdef GKFS_USE_GUIDED_DISTRIBUTION auto distributor = std::make_shared( CTX->local_host_id(), CTX->hosts().size()); +#elif GKFS_ENABLE_EC + auto distributor = std::make_shared( + CTX->local_host_id(), CTX->hosts().size(), CTX->get_replicas()); #else auto distributor = std::make_shared( CTX->local_host_id(), CTX->hosts().size()); @@ -238,6 +241,9 @@ init_environment() { EXIT_FAILURE, "Unable to fetch file system configurations from daemon process through RPC."); } + if(CTX->get_replicas() > 0) { + srand(time(NULL)); + } LOG(INFO, "Environment initialization successful."); } @@ -275,6 +281,7 @@ init_preload() { gkfs::path::init_cwd(); LOG(DEBUG, "Current working directory: '{}'", CTX->cwd()); + LOG(DEBUG, "Number of replicas : '{}'", CTX->get_replicas()); gkfs::preload::init_environment(); CTX->enable_interception(); diff --git a/src/client/preload_context.cpp b/src/client/preload_context.cpp index 236d2583f8dda978f3ad65e6e8c6c5815cc205ad..d538cd5ec573d6eb0ac9d196d806627552207ac3 100644 --- a/src/client/preload_context.cpp +++ b/src/client/preload_context.cpp @@ -65,6 +65,11 @@ PreloadContext::PreloadContext() char host[255]; gethostname(host, 255); hostname = host; + PreloadContext::set_replicas( + std::stoi(gkfs::env::get_var(gkfs::env::NUM_REPL, "0"))); + + PreloadContext::set_ec_ondemand( + std::stoi(gkfs::env::get_var(gkfs::env::EC_ONDEMAND, "0")) == 1); } void @@ -447,5 +452,26 @@ PreloadContext::get_hostname() { return hostname; } +void +PreloadContext::set_replicas(const int repl) { + replicas_ = repl; +} + +int +PreloadContext::get_replicas() { + return replicas_; +} + +void +PreloadContext::set_ec_ondemand(const bool ec_ondemand) { + ec_ondemand_ = ec_ondemand; +} + +bool +PreloadContext::get_ec_ondemand() { + return ec_ondemand_; +} + + } // namespace preload } // namespace gkfs diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index 2a77c37a36f2824f3f74b5bc73efbf55dd6247c3..4a9c2b1c4be2c0c0d5ea9e83dc8b01e538d1b03e 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -200,16 +200,26 @@ namespace gkfs::utils { optional get_metadata(const string& path, bool follow_links) { std::string attr; - auto err = gkfs::rpc::forward_stat(path, attr); + auto err = gkfs::rpc::forward_stat(path, attr, 0); + // TODO: retry on failure + if(err) { - errno = err; - return {}; + auto copy = 1; + while(copy < CTX->get_replicas() + 1 && err) { + LOG(ERROR, "Retrying Stat on replica {} {}", copy, follow_links); + err = gkfs::rpc::forward_stat(path, attr, copy); + copy++; + } + if(err) { + errno = err; + return {}; + } } #ifdef HAS_SYMLINKS if(follow_links) { gkfs::metadata::Metadata md{attr}; while(md.is_link()) { - err = gkfs::rpc::forward_stat(md.target_path(), attr); + err = gkfs::rpc::forward_stat(md.target_path(), attr, 0); if(err) { errno = err; return {}; diff --git a/src/client/rpc/forward_data.cpp b/src/client/rpc/forward_data.cpp index 93d72efcdaba4d250b42f71398b5510a2c590534..a400af25f621b25e4e6a0e77541df93143d56220 100644 --- a/src/client/rpc/forward_data.cpp +++ b/src/client/rpc/forward_data.cpp @@ -34,9 +34,15 @@ #include #include +#include #include +#ifdef GKFS_ENABLE_EC +#include +#include +#endif + using namespace std; namespace gkfs::rpc { @@ -46,17 +52,299 @@ namespace gkfs::rpc { * NOTE: No errno is defined here! */ +/** + * @brief Calculate the chunk start and end that will be affected by the + * operation. + * + * @param path + * @param append_flag + * @param in_offset + * @param write_size + * @param updated_metadentry_size + * @param num_copies + * @return pair + */ +std::pair +calc_op_chunks(const std::string& path, const bool append_flag, + const off64_t in_offset, const size_t write_size, + const int64_t updated_metadentry_size) { + using namespace gkfs::utils::arithmetic; + off64_t offset = + append_flag ? in_offset : (updated_metadentry_size - write_size); + + auto chnk_start = block_index(offset, gkfs::config::rpc::chunksize); + auto chnk_end = block_index((offset + write_size) - 1, + gkfs::config::rpc::chunksize); + + + return make_pair(chnk_start, chnk_end); +} + +// #ifdef GKFS_ENABLE_EC +/** + * Send an RPC request to write a single chunk of data to a given server + * @param path + * @param buf + * @param write_size + * @param server destination server + * @return pair + */ +pair +ecc_forward_write(const string& path, const void* buf, const size_t write_size, + const int8_t server) { + + // import pow2-optimized arithmetic functions + using namespace gkfs::utils::arithmetic; + + assert(write_size > 0); + + std::vector write_ops_vect(8); + gkfs::rpc::set_bitset(write_ops_vect, 0); + + // some helper variables for async RPC + std::vector bufseq{ + hermes::mutable_buffer{const_cast(buf), write_size}, + }; + + // expose user buffers so that they can serve as RDMA data sources + // (these are automatically "unexposed" when the destructor is called) + hermes::exposed_memory local_buffers; + + try { + local_buffers = ld_network_service->expose( + bufseq, hermes::access_mode::read_only); + + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to expose buffers for RMA"); + return make_pair(EBUSY, 0); + } + + std::vector> handles; + + // Issue non-blocking RPC requests and wait for the result later + // + // TODO(amiranda): This could be simplified by adding a vector of inputs + // to async_engine::broadcast(). This would allow us to avoid manually + // looping over handles as we do below + auto target = server; + + // total chunk_size for target + auto total_chunk_size = gkfs::config::rpc::chunksize; + + auto endp = CTX->hosts().at(target); + + try { + LOG(DEBUG, "Sending RPC ..."); + + gkfs::rpc::write_data::input in( + path, + // first offset in targets is the chunk with + // a potential offset + 0, target, CTX->hosts().size(), + // number of chunks handled by that destination + gkfs::rpc::compressBitset(write_ops_vect), 1, + // chunk start id of this write + 0, + // chunk end id of this write + 0, + // total size to write + total_chunk_size, local_buffers); + + // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that + // we can retry for RPC_TRIES (see old commits with margo) + // TODO(amiranda): hermes will eventually provide a post(endpoint) + // returning one result and a broadcast(endpoint_set) returning a + // result_set. When that happens we can remove the .at(0) :/ + handles.emplace_back( + ld_network_service->post(endp, in)); + + LOG(DEBUG, "host: {}, path: \"{}\", size: {}", target, path, + total_chunk_size); + } catch(const std::exception& ex) { + LOG(ERROR, + "Unable to send non-blocking rpc for " + "path \"{}\" [peer: {}]", + path, target); + + return make_pair(EBUSY, 0); + } + + auto err = 0; + ssize_t out_size = 0; + std::size_t idx = 0; + + for(const auto& h : handles) { + try { + // XXX We might need a timeout here to not wait forever for an + // output that never comes? + auto out = h.get().at(0); + + if(out.err() != 0) { + LOG(ERROR, "Daemon reported error: {}", out.err()); + err = out.err(); + } else { + out_size += static_cast(out.io_size()); + } + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to get rpc output for path \"{}\"", path); + err = EIO; + } + idx++; + } + + /* + * Typically file systems return the size even if only a part of it was + * written. In our case, we do not keep track which daemon fully wrote its + * workload. Thus, we always return size 0 on error. + */ + if(err) + return make_pair(err, 0); + else + return make_pair(0, out_size); +} +/** + * Send an RPC request to read to a buffer. + * @param path + * @param buf + * @param offset + * @param read_size + * @param num_copies number of copies available (0 is no replication) + * @param failed nodes failed that should not be used + * @return pair + */ +pair +ecc_forward_read(const string& path, void* buf, const size_t read_size, + const int8_t server) { + + + std::vector read_bitset_vect(8, 0); + gkfs::rpc::set_bitset(read_bitset_vect, 0); + + // some helper variables for async RPCs + std::vector bufseq{ + hermes::mutable_buffer{buf, read_size}, + }; + + // expose user buffers so that they can serve as RDMA data targets + // (these are automatically "unexposed" when the destructor is called) + hermes::exposed_memory local_buffers; + + try { + local_buffers = ld_network_service->expose( + bufseq, hermes::access_mode::write_only); + + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to expose buffers for RMA"); + return make_pair(EBUSY, 0); + } + + std::vector> handles; + + // Issue non-blocking RPC requests and wait for the result later + // + // TODO(amiranda): This could be simplified by adding a vector of inputs + // to async_engine::broadcast(). This would allow us to avoid manually + // looping over handles as we do below + + auto target = server; + + auto endp = CTX->hosts().at(target); + + try { + + LOG(DEBUG, "Sending RPC ..."); + + gkfs::rpc::read_data::input in( + path, + // first offset in targets is the chunk with + // a potential offset + 0, target, CTX->hosts().size(), + gkfs::rpc::compressBitset(read_bitset_vect), + // number of chunks handled by that destination + 1, + // chunk start id of this write + 0, + // chunk end id of this write + 0, + // total size to write + read_size, local_buffers); + + // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so + // that we can retry for RPC_TRIES (see old commits with margo) + // TODO(amiranda): hermes will eventually provide a + // post(endpoint) returning one result and a + // broadcast(endpoint_set) returning a result_set. When that + // happens we can remove the .at(0) :/ + handles.emplace_back( + ld_network_service->post(endp, in)); + + } catch(const std::exception& ex) { + LOG(ERROR, + "Unable to send non-blocking rpc for path \"{}\" " + "[peer: {}]", + path, target); + return make_pair(EBUSY, 0); + } + + // Wait for RPC responses and then get response and add it to out_size + // which is the read size. All potential outputs are served to free + // resources regardless of errors, although an errorcode is set. + auto err = 0; + ssize_t out_size = 0; + std::size_t idx = 0; + + for(const auto& h : handles) { + try { + // XXX We might need a timeout here to not wait forever for an + // output that never comes? + auto out = h.get().at(0); + + if(out.err() != 0) { + LOG(ERROR, "Daemon reported error: {}", out.err()); + err = out.err(); + } + + out_size += static_cast(out.io_size()); + + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to get rpc output for path \"{}\" [peer: {}]", + path, target); + err = EIO; + } + idx++; + } + + + /* + * Typically file systems return the size even if only a part of it was + * read. In our case, we do not keep track which daemon fully read its + * workload. Thus, we always return size 0 on error. + */ + if(err) + return make_pair(err, 0); + else + return make_pair(0, out_size); +} + +// #endif + /** * Send an RPC request to write from a buffer. + * There is a bitset of 1024 chunks to tell the server + * which chunks to process. Exceeding this value will work without + * replication. Another way is to leverage mercury segments. + * TODO: Decide how to manage a write to a replica that doesn't exist * @param path * @param buf * @param append_flag * @param write_size + * @param updated_metadentry_size + * @param num_copies number of replicas * @return pair */ pair forward_write(const string& path, const void* buf, const off64_t offset, - const size_t write_size) { + const size_t write_size, const int8_t num_copies) { // import pow2-optimized arithmetic functions using namespace gkfs::utils::arithmetic; @@ -69,35 +357,50 @@ forward_write(const string& path, const void* buf, const off64_t offset, auto chnk_end = block_index((offset + write_size) - 1, gkfs::config::rpc::chunksize); + auto chnk_total = (chnk_end - chnk_start) + 1; + // Collect all chunk ids within count that have the same destination so // that those are send in one rpc bulk transfer std::map> target_chnks{}; + // contains the target ids, used to access the target_chnks map. // First idx is chunk with potential offset std::vector targets{}; // targets for the first and last chunk as they need special treatment - uint64_t chnk_start_target = 0; - uint64_t chnk_end_target = 0; + // We need a set to manage replicas. + std::set chnk_start_target{}; + std::set chnk_end_target{}; - for(uint64_t chnk_id = chnk_start; chnk_id <= chnk_end; chnk_id++) { - auto target = CTX->distributor()->locate_data(path, chnk_id); + std::unordered_map> write_ops_vect; - if(target_chnks.count(target) == 0) { - target_chnks.insert( - std::make_pair(target, std::vector{chnk_id})); - targets.push_back(target); - } else { - target_chnks[target].push_back(chnk_id); - } + // If num_copies is 0, we do the normal write operation. Otherwise + // we process all the replicas. + for(uint64_t chnk_id = chnk_start; chnk_id <= chnk_end; chnk_id++) { + for(auto copy = num_copies ? 1 : 0; copy < num_copies + 1; copy++) { + auto target = CTX->distributor()->locate_data(path, chnk_id, copy); + + if(write_ops_vect.find(target) == write_ops_vect.end()) + write_ops_vect[target] = + std::vector(((chnk_total + 7) / 8)); + gkfs::rpc::set_bitset(write_ops_vect[target], chnk_id - chnk_start); + + if(target_chnks.count(target) == 0) { + target_chnks.insert( + std::make_pair(target, std::vector{chnk_id})); + targets.push_back(target); + } else { + target_chnks[target].push_back(chnk_id); + } - // set first and last chnk targets - if(chnk_id == chnk_start) { - chnk_start_target = target; - } + // set first and last chnk targets + if(chnk_id == chnk_start) { + chnk_start_target.insert(target); + } - if(chnk_id == chnk_end) { - chnk_end_target = target; + if(chnk_id == chnk_end) { + chnk_end_target.insert(target); + } } } @@ -133,13 +436,13 @@ forward_write(const string& path, const void* buf, const off64_t offset, target_chnks[target].size() * gkfs::config::rpc::chunksize; // receiver of first chunk must subtract the offset from first chunk - if(target == chnk_start_target) { + if(chnk_start_target.end() != chnk_start_target.find(target)) { total_chunk_size -= block_overrun(offset, gkfs::config::rpc::chunksize); } // receiver of last chunk must subtract - if(target == chnk_end_target && + if(chnk_end_target.end() != chnk_end_target.find(target) && !is_aligned(offset + write_size, gkfs::config::rpc::chunksize)) { total_chunk_size -= block_underrun(offset + write_size, gkfs::config::rpc::chunksize); @@ -148,7 +451,6 @@ forward_write(const string& path, const void* buf, const off64_t offset, auto endp = CTX->hosts().at(target); try { - LOG(DEBUG, "Sending RPC ..."); gkfs::rpc::write_data::input in( @@ -158,6 +460,7 @@ forward_write(const string& path, const void* buf, const off64_t offset, block_overrun(offset, gkfs::config::rpc::chunksize), target, CTX->hosts().size(), // number of chunks handled by that destination + gkfs::rpc::compressBitset(write_ops_vect[target]), target_chnks[target].size(), // chunk start id of this write chnk_start, @@ -175,25 +478,26 @@ forward_write(const string& path, const void* buf, const off64_t offset, ld_network_service->post(endp, in)); LOG(DEBUG, - "host: {}, path: \"{}\", chunks: {}, size: {}, offset: {}", - target, path, in.chunk_n(), total_chunk_size, in.offset()); - + "host: {}, path: \"{}\", chunk_start: {}, chunk_end: {}, chunks: {}, size: {}, offset: {}", + target, path, chnk_start, chnk_end, in.chunk_n(), + total_chunk_size, in.offset()); } catch(const std::exception& ex) { LOG(ERROR, "Unable to send non-blocking rpc for " "path \"{}\" [peer: {}]", path, target); - return make_pair(EBUSY, 0); + if(num_copies == 0) + return make_pair(EBUSY, 0); } } - // Wait for RPC responses and then get response and add it to out_size - // which is the written size All potential outputs are served to free - // resources regardless of errors, although an errorcode is set. auto err = 0; ssize_t out_size = 0; std::size_t idx = 0; - +#ifdef REPLICA_CHECK + std::vector fill(chnk_total); + auto write_ops = write_ops_vect.begin(); +#endif for(const auto& h : handles) { try { // XXX We might need a timeout here to not wait forever for an @@ -203,18 +507,52 @@ forward_write(const string& path, const void* buf, const off64_t offset, if(out.err() != 0) { LOG(ERROR, "Daemon reported error: {}", out.err()); err = out.err(); + } else { + out_size += static_cast(out.io_size()); +#ifdef REPLICA_CHECK + if(num_copies) { + if(fill.size() == 0) { + fill = write_ops->second; + } else { + for(size_t i = 0; i < fill.size(); i++) { + fill[i] |= write_ops->second[i]; + } + } + } + write_ops++; +#endif } - - out_size += static_cast(out.io_size()); - } catch(const std::exception& ex) { LOG(ERROR, "Failed to get rpc output for path \"{}\" [peer: {}]", path, targets[idx]); err = EIO; } - idx++; } + // As servers can fail (and we cannot know if the total data is written), we + // send the updated size but check that at least one copy of all chunks are + // processed. + if(num_copies) { + // A bit-wise or should show that all the chunks are written (255) + out_size = write_size; +#ifdef REPLICA_CHECK + for(size_t i = 0; i < fill.size() - 1; i++) { + if(fill[i] != 255) { + err = EIO; + break; + } + } + // Process the leftover bytes + for(uint64_t chnk_id = (chnk_start + (fill.size() - 1) * 8); + chnk_id <= chnk_end; chnk_id++) { + if(!(fill[(chnk_id - chnk_start) / 8] & + (1 << ((chnk_id - chnk_start) % 8)))) { + err = EIO; + break; + } + } +#endif + } /* * Typically file systems return the size even if only a part of it was * written. In our case, we do not keep track which daemon fully wrote its @@ -226,17 +564,128 @@ forward_write(const string& path, const void* buf, const off64_t offset, return make_pair(0, out_size); } +#ifdef GKFS_ENABLE_EC +/** + * @brief process a chunk line to recover a missing server chunk, can be used + * for multiple failures + * + * @param path + * @param buffer_recover + * @param chunk_candidate + * @param failed_server hint to indicate whether the server chunk was corrupted + * (not necessarily) + * @return true + * @return false + */ +bool +gkfs_ecc_recover(const std::string& path, void* buffer_recover, + uint64_t chunk_candidate, uint64_t failed_server) { + + + auto data_servers = CTX->hosts().size() - CTX->get_replicas(); + + char** data = (char**) malloc(sizeof(char*) * data_servers); + char** coding = (char**) malloc(sizeof(char*) * CTX->get_replicas()); + + for(unsigned int i = 0; i < data_servers; ++i) { + data[i] = (char*) malloc(gkfs::config::rpc::chunksize); + } + for(auto i = 0; i < CTX->get_replicas(); ++i) { + coding[i] = (char*) malloc(gkfs::config::rpc::chunksize); + } + + auto initial_row_chunk = (chunk_candidate / data_servers) * data_servers; + + + // 1 - Read data from the other chunks (failures allowed) + + LOG(DEBUG, "Operation Size - Range {} - data_servers {} replica_servers {}", + initial_row_chunk, data_servers, CTX->get_replicas()); + + vector erased; + + auto i = initial_row_chunk; + + for(uint64_t j = 0; j < data_servers; ++j) { + std::set failed; + LOG(DEBUG, "Reading Chunk {} -> {}, from server {}", i, i + j, j); + // We set num_replicas to 0, to avoid recursion issues + // We only want to read, no recover and we could do it without loop... + auto out = gkfs::rpc::forward_read( + path, data[j], (j + i) * gkfs::config::rpc::chunksize, + gkfs::config::rpc::chunksize, 0, failed); + if(out.first != 0) { + LOG(ERROR, "Read Parity Error: {}", out.first); + erased.push_back(j); + } + } + + // Read ec codes, TODO: Delete this file once the original is gone or the + // file shrinks + std::string ecc_path = path + "_ecc_" + to_string(i) + "_" + + to_string(i + data_servers - 1); + + for(auto ecc_num = 0; ecc_num < CTX->get_replicas(); ++ecc_num) { + LOG(DEBUG, "Reading EC Chunk {} from server {} ", ecc_path, + ecc_num + data_servers); + auto out = gkfs::rpc::ecc_forward_read(ecc_path, coding[ecc_num], + gkfs::config::rpc::chunksize, + ecc_num + data_servers); + + if(out.first != 0) { + LOG(ERROR, "Read Parity Error: {}", out.first); + erased.push_back(ecc_num + data_servers); + } else { + LOG(DEBUG, "Read EC Success"); + } + } + + // We force a failure + erased.push_back(failed_server); + erased.push_back(-1); + + + int res = 0; + + // We have all the data to recover the buffer + int* matrix = reed_sol_vandermonde_coding_matrix(data_servers, + CTX->get_replicas(), 8); + + res = jerasure_matrix_decode(data_servers, CTX->get_replicas(), 8, matrix, + 0, erased.data(), data, coding, + gkfs::config::rpc::chunksize); + + free(matrix); + LOG(DEBUG, "EC recovered {}, with result {}", failed_server, res); + memcpy(buffer_recover, data[failed_server], gkfs::config::rpc::chunksize); + + LOG(DEBUG, "EC computation finished"); + for(unsigned int i = 0; i < data_servers; ++i) { + free(data[i]); + } + for(auto i = 0; i < CTX->get_replicas(); ++i) { + free(coding[i]); + } + free(data); + free(coding); + return true; +} +#endif + /** * Send an RPC request to read to a buffer. * @param path * @param buf * @param offset * @param read_size + * @param num_copies number of copies available (0 is no replication) + * @param failed nodes failed that should not be used * @return pair */ pair forward_read(const string& path, void* buf, const off64_t offset, - const size_t read_size) { + const size_t read_size, const int8_t num_copies, + std::set& failed) { // import pow2-optimized arithmetic functions using namespace gkfs::utils::arithmetic; @@ -246,19 +695,34 @@ forward_read(const string& path, void* buf, const off64_t offset, auto chnk_start = block_index(offset, gkfs::config::rpc::chunksize); auto chnk_end = block_index((offset + read_size - 1), gkfs::config::rpc::chunksize); - + auto chnk_total = (chnk_end - chnk_start) + 1; // Collect all chunk ids within count that have the same destination so // that those are send in one rpc bulk transfer std::map> target_chnks{}; + // contains the recipient ids, used to access the target_chnks map. // First idx is chunk with potential offset std::vector targets{}; // targets for the first and last chunk as they need special treatment uint64_t chnk_start_target = 0; uint64_t chnk_end_target = 0; + std::unordered_map> read_bitset_vect; for(uint64_t chnk_id = chnk_start; chnk_id <= chnk_end; chnk_id++) { - auto target = CTX->distributor()->locate_data(path, chnk_id); + auto target = CTX->distributor()->locate_data(path, chnk_id, 0); + if(num_copies > 0) { + // If we have some failures we select another copy (randomly). + while(failed.find(target) != failed.end()) { + LOG(DEBUG, "Selecting another node, target: {} down", target); + target = CTX->distributor()->locate_data(path, chnk_id, + rand() % num_copies); + } + } + if(read_bitset_vect.find(target) == read_bitset_vect.end()) + read_bitset_vect[target] = + std::vector(((chnk_total + 7) / 8)); + + gkfs::rpc::set_bitset(read_bitset_vect[target], chnk_id - chnk_start); if(target_chnks.count(target) == 0) { target_chnks.insert( @@ -303,6 +767,7 @@ forward_read(const string& path, void* buf, const off64_t offset, // TODO(amiranda): This could be simplified by adding a vector of inputs // to async_engine::broadcast(). This would allow us to avoid manually // looping over handles as we do below + for(const auto& target : targets) { // total chunk_size for target @@ -334,6 +799,7 @@ forward_read(const string& path, void* buf, const off64_t offset, // a potential offset block_overrun(offset, gkfs::config::rpc::chunksize), target, CTX->hosts().size(), + gkfs::rpc::compressBitset(read_bitset_vect[target]), // number of chunks handled by that destination target_chnks[target].size(), // chunk start id of this write @@ -343,11 +809,12 @@ forward_read(const string& path, void* buf, const off64_t offset, // total size to write total_chunk_size, local_buffers); - // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that - // we can retry for RPC_TRIES (see old commits with margo) - // TODO(amiranda): hermes will eventually provide a post(endpoint) - // returning one result and a broadcast(endpoint_set) returning a - // result_set. When that happens we can remove the .at(0) :/ + // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so + // that we can retry for RPC_TRIES (see old commits with margo) + // TODO(amiranda): hermes will eventually provide a + // post(endpoint) returning one result and a + // broadcast(endpoint_set) returning a result_set. When that + // happens we can remove the .at(0) :/ handles.emplace_back( ld_network_service->post(endp, in)); @@ -387,16 +854,106 @@ forward_read(const string& path, void* buf, const off64_t offset, LOG(ERROR, "Daemon reported error: {}", out.err()); err = out.err(); } - +#ifdef GKFS_ENABLE_READ_ERRORS + if(rand() % 2 == 0 and num_copies > 0) { + throw std::exception(); + } +#endif // GKFS_ENABLE_READ_ERRORS out_size += static_cast(out.io_size()); } catch(const std::exception& ex) { LOG(ERROR, "Failed to get rpc output for path \"{}\" [peer: {}]", path, targets[idx]); err = EIO; + // We should get targets[idx] and remove from the list of peers + failed.insert(targets[idx]); + // Then repeat the read with another peer (We repear the full + // read, this can be optimised but it is a cornercase) + +#ifdef GKFS_ENABLE_EC + // We try to recover the missing data from the failed server + // obtain the full chunk from all the other servers + // Decode the data + // Fill the gaps, and then remove the failed server while + // keeping the variables consistent. + if(num_copies > 0) { + auto failed_server = targets[idx]; + + // For all the chunks activated in the bitset, recover and fill + // the buffer. + for(auto chnk_id_file = chnk_start; chnk_id_file <= chnk_end; + chnk_id_file++) { + // Continue if chunk does not hash to this host + // We only check if we are not using replicas + + if(!(gkfs::rpc::get_bitset(read_bitset_vect[failed_server], + chnk_id_file - chnk_start))) { + + continue; + } + + // We have a chunk to recover + // We don't need to worry about offset etc... just use the + // chunk number + void* recovered_chunk = + malloc(gkfs::config::rpc::chunksize); + auto recovered = gkfs::rpc::gkfs_ecc_recover( + path, recovered_chunk, chnk_id_file, failed_server); + LOG(DEBUG, "Recovered server: {} Result {}", failed_server, + recovered); + + // Move recovered_chunk to the buffer, first and last chunk + // should substract... + auto recover_size = gkfs::config::rpc::chunksize; + auto recover_offt = (chnk_id_file - chnk_start) * + gkfs::config::rpc::chunksize; + auto recover_offt_chunk = 0; + + if(chnk_id_file == chnk_start) { + // We may need to move the offset of both buffers and + // reduce the recover size + auto offset_fc = block_overrun( + offset, gkfs::config::rpc::chunksize); + recover_offt += offset_fc; + recover_offt_chunk += offset_fc; + recover_size -= offset_fc; + } + if(chnk_id_file == chnk_end) { + // We may need to reduce the recover size. + if(!is_aligned(offset + read_size, + gkfs::config::rpc::chunksize)) { + recover_size -= block_underrun( + offset + read_size, + gkfs::config::rpc::chunksize); + } + } + LOG(DEBUG, + "Recovered chunk : Start Offset {}/OffsetChunk {} - Size {}", + recover_offt, recover_offt_chunk, recover_size); + + if(recovered) { + err = 0; + out_size += static_cast(recover_size); + } else { + err = EIO; + out_size = -1; + LOG(ERROR, "Can't recover error with ec"); + } + + + memcpy((char*) buf + recover_offt, + (char*) recovered_chunk + recover_offt_chunk, + recover_size); + + free(recovered_chunk); + } + } + +#endif } idx++; } + /* * Typically file systems return the size even if only a part of it was * read. In our case, we do not keep track which daemon fully read its @@ -413,11 +970,12 @@ forward_read(const string& path, void* buf, const off64_t offset, * @param path * @param current_size * @param new_size + * @param num_copies Number of replicas * @return error code */ int -forward_truncate(const std::string& path, size_t current_size, - size_t new_size) { +forward_truncate(const std::string& path, size_t current_size, size_t new_size, + const int8_t num_copies) { // import pow2-optimized arithmetic functions using namespace gkfs::utils::arithmetic; @@ -434,7 +992,9 @@ forward_truncate(const std::string& path, size_t current_size, std::unordered_set hosts; for(unsigned int chunk_id = chunk_start; chunk_id <= chunk_end; ++chunk_id) { - hosts.insert(CTX->distributor()->locate_data(path, chunk_id)); + for(auto copy = 0; copy < (num_copies + 1); ++copy) { + hosts.insert(CTX->distributor()->locate_data(path, chunk_id, copy)); + } } std::vector> handles; @@ -450,20 +1010,23 @@ forward_truncate(const std::string& path, size_t current_size, gkfs::rpc::trunc_data::input in(path, new_size); - // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that - // we can retry for RPC_TRIES (see old commits with margo) - // TODO(amiranda): hermes will eventually provide a post(endpoint) - // returning one result and a broadcast(endpoint_set) returning a - // result_set. When that happens we can remove the .at(0) :/ + // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so + // that we can retry for RPC_TRIES (see old commits with margo) + // TODO(amiranda): hermes will eventually provide a + // post(endpoint) returning one result and a + // broadcast(endpoint_set) returning a result_set. When that + // happens we can remove the .at(0) :/ handles.emplace_back( ld_network_service->post(endp, in)); } catch(const std::exception& ex) { - // TODO(amiranda): we should cancel all previously posted requests - // here, unfortunately, Hermes does not support it yet :/ + // TODO(amiranda): we should cancel all previously posted + // requests here, unfortunately, Hermes does not support it yet + // :/ LOG(ERROR, "Failed to send request to host: {}", host); err = EIO; - break; // We need to gather all responses so we can't return here + break; // We need to gather all responses so we can't return + // here } } @@ -503,20 +1066,23 @@ forward_get_chunk_stat() { gkfs::rpc::chunk_stat::input in(0); - // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that - // we can retry for RPC_TRIES (see old commits with margo) - // TODO(amiranda): hermes will eventually provide a post(endpoint) - // returning one result and a broadcast(endpoint_set) returning a - // result_set. When that happens we can remove the .at(0) :/ + // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so + // that we can retry for RPC_TRIES (see old commits with margo) + // TODO(amiranda): hermes will eventually provide a + // post(endpoint) returning one result and a + // broadcast(endpoint_set) returning a result_set. When that + // happens we can remove the .at(0) :/ handles.emplace_back( ld_network_service->post(endp, in)); } catch(const std::exception& ex) { - // TODO(amiranda): we should cancel all previously posted requests - // here, unfortunately, Hermes does not support it yet :/ + // TODO(amiranda): we should cancel all previously posted + // requests here, unfortunately, Hermes does not support it yet + // :/ LOG(ERROR, "Failed to send request to host: {}", endp.to_string()); err = EBUSY; - break; // We need to gather all responses so we can't return here + break; // We need to gather all responses so we can't return + // here } } @@ -547,9 +1113,11 @@ forward_get_chunk_stat() { chunk_free += out.chunk_free(); } catch(const std::exception& ex) { LOG(ERROR, "Failed to get RPC output from host: {}", i); - err = EBUSY; + // Avoid setting err if a server fails. + // err = EBUSY; } } + if(err) return make_pair(err, ChunkStat{}); else diff --git a/src/client/rpc/forward_management.cpp b/src/client/rpc/forward_management.cpp index 4ebbef9b35bd11320b657222bd938182f1e2daf8..a44b37a1eba15f03a3da3387c188881294efe9b9 100644 --- a/src/client/rpc/forward_management.cpp +++ b/src/client/rpc/forward_management.cpp @@ -45,19 +45,30 @@ forward_get_fs_config() { auto endp = CTX->hosts().at(CTX->local_host_id()); gkfs::rpc::fs_config::output out; - try { - LOG(DEBUG, "Retrieving file system configurations from daemon"); - // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that we - // can retry for RPC_TRIES (see old commits with margo) - // TODO(amiranda): hermes will eventually provide a post(endpoint) - // returning one result and a broadcast(endpoint_set) returning a - // result_set. When that happens we can remove the .at(0) :/ - out = ld_network_service->post(endp).get().at(0); - } catch(const std::exception& ex) { - LOG(ERROR, "Retrieving fs configurations from daemon"); - return false; + bool found = false; + size_t idx = 0; + while(!found && idx <= CTX->hosts().size()) { + try { + LOG(DEBUG, "Retrieving file system configurations from daemon"); + // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that + // we can retry for RPC_TRIES (see old commits with margo) + // TODO(amiranda): hermes will eventually provide a post(endpoint) + // returning one result and a broadcast(endpoint_set) returning a + // result_set. When that happens we can remove the .at(0) :/ + out = ld_network_service->post(endp).get().at( + 0); + found = true; + } catch(const std::exception& ex) { + LOG(ERROR, + "Retrieving fs configurations from daemon, possible reattempt at peer: {}", + idx); + endp = CTX->hosts().at(idx++); + } } + if(!found) + return false; + CTX->mountdir(out.mountdir()); LOG(INFO, "Mountdir: '{}'", CTX->mountdir()); diff --git a/src/client/rpc/forward_metadata.cpp b/src/client/rpc/forward_metadata.cpp index a087590c807acfe33697775931f816c8f58cddd5..407bc484f45fc04c5f05b1642e68efcb3640e527 100644 --- a/src/client/rpc/forward_metadata.cpp +++ b/src/client/rpc/forward_metadata.cpp @@ -51,12 +51,14 @@ namespace gkfs::rpc { * Send an RPC for a create request * @param path * @param mode + * @param copy Number of replica to create * @return error code */ int -forward_create(const std::string& path, const mode_t mode) { +forward_create(const std::string& path, const mode_t mode, const int copy) { - auto endp = CTX->hosts().at(CTX->distributor()->locate_file_metadata(path)); + auto endp = CTX->hosts().at( + CTX->distributor()->locate_file_metadata(path, copy)); try { LOG(DEBUG, "Sending RPC ..."); @@ -81,12 +83,14 @@ forward_create(const std::string& path, const mode_t mode) { * Send an RPC for a stat request * @param path * @param attr + * @param copy metadata replica to read from * @return error code */ int -forward_stat(const std::string& path, string& attr) { +forward_stat(const std::string& path, string& attr, const int copy) { - auto endp = CTX->hosts().at(CTX->distributor()->locate_file_metadata(path)); + auto endp = CTX->hosts().at( + CTX->distributor()->locate_file_metadata(path, copy)); try { LOG(DEBUG, "Sending RPC ..."); @@ -121,40 +125,44 @@ forward_stat(const std::string& path, string& attr) { * This function only attempts data removal if data exists (determined when * metadata is removed) * @param path + * @param num_copies Replication scenarios with many replicas * @return error code */ int -forward_remove(const std::string& path) { - - auto endp = CTX->hosts().at(CTX->distributor()->locate_file_metadata(path)); +forward_remove(const std::string& path, const int8_t num_copies) { int64_t size = 0; uint32_t mode = 0; - /* - * Send one RPC to metadata destination and remove metadata while retrieving - * size and mode to determine if data needs to removed too - */ - try { - LOG(DEBUG, "Sending RPC ..."); - // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that we - // can retry for RPC_TRIES (see old commits with margo) - // TODO(amiranda): hermes will eventually provide a post(endpoint) - // returning one result and a broadcast(endpoint_set) returning a - // result_set. When that happens we can remove the .at(0) :/ - auto out = - ld_network_service->post(endp, path) - .get() - .at(0); - - LOG(DEBUG, "Got response success: {}", out.err()); + for(auto copy = 0; copy < (num_copies + 1); copy++) { + auto endp = CTX->hosts().at( + CTX->distributor()->locate_file_metadata(path, copy)); - if(out.err()) - return out.err(); - size = out.size(); - mode = out.mode(); - } catch(const std::exception& ex) { - LOG(ERROR, "while getting rpc output"); - return EBUSY; + /* + * Send one RPC to metadata destination and remove metadata while + * retrieving size and mode to determine if data needs to removed too + */ + try { + LOG(DEBUG, "Sending RPC ..."); + // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that + // we can retry for RPC_TRIES (see old commits with margo) + // TODO(amiranda): hermes will eventually provide a post(endpoint) + // returning one result and a broadcast(endpoint_set) returning a + // result_set. When that happens we can remove the .at(0) :/ + auto out = ld_network_service + ->post(endp, path) + .get() + .at(0); + + LOG(DEBUG, "Got response success: {}", out.err()); + + if(out.err()) + return out.err(); + size = out.size(); + mode = out.mode(); + } catch(const std::exception& ex) { + LOG(ERROR, "while getting rpc output"); + return EBUSY; + } } // if file is not a regular file and it's size is 0, data does not need to // be removed, thus, we exit @@ -167,44 +175,54 @@ forward_remove(const std::string& path) { // Small files if(static_cast(size / gkfs::config::rpc::chunksize) < CTX->hosts().size()) { - const auto metadata_host_id = - CTX->distributor()->locate_file_metadata(path); - const auto endp_metadata = CTX->hosts().at(metadata_host_id); - - try { - LOG(DEBUG, "Sending RPC to host: {}", endp_metadata.to_string()); - gkfs::rpc::remove_data::input in(path); - handles.emplace_back( - ld_network_service->post( - endp_metadata, in)); - - uint64_t chnk_start = 0; - uint64_t chnk_end = size / gkfs::config::rpc::chunksize; - - for(uint64_t chnk_id = chnk_start; chnk_id <= chnk_end; chnk_id++) { - const auto chnk_host_id = - CTX->distributor()->locate_data(path, chnk_id); - if constexpr(gkfs::config::metadata::implicit_data_removal) { - /* - * If the chnk host matches the metadata host the remove - * request as already been sent as part of the metadata - * remove request. - */ - if(chnk_host_id == metadata_host_id) - continue; - } - const auto endp_chnk = CTX->hosts().at(chnk_host_id); - - LOG(DEBUG, "Sending RPC to host: {}", endp_chnk.to_string()); + for(auto copymd = 0; copymd < (num_copies + 1); copymd++) { + const auto metadata_host_id = + CTX->distributor()->locate_file_metadata(path, copymd); + const auto endp_metadata = CTX->hosts().at(metadata_host_id); + try { + LOG(DEBUG, "Sending RPC to host: {}", + endp_metadata.to_string()); + gkfs::rpc::remove_data::input in(path); handles.emplace_back( ld_network_service->post( - endp_chnk, in)); + endp_metadata, in)); + + uint64_t chnk_start = 0; + uint64_t chnk_end = size / gkfs::config::rpc::chunksize; + + for(uint64_t chnk_id = chnk_start; chnk_id <= chnk_end; + chnk_id++) { + for(auto copy = 0; copy < (num_copies + 1); copy++) { + const auto chnk_host_id = + CTX->distributor()->locate_data(path, chnk_id, + copy); + if constexpr(gkfs::config::metadata:: + implicit_data_removal) { + /* + * If the chnk host matches the metadata host the + * remove request as already been sent as part of + * the metadata remove request. + */ + if(chnk_host_id == metadata_host_id) + continue; + } + const auto endp_chnk = CTX->hosts().at(chnk_host_id); + + LOG(DEBUG, "Sending RPC to host: {}", + endp_chnk.to_string()); + + handles.emplace_back( + ld_network_service + ->post( + endp_chnk, in)); + } + } + } catch(const std::exception& ex) { + LOG(ERROR, + "Failed to forward non-blocking rpc request reduced remove requests"); + return EBUSY; } - } catch(const std::exception& ex) { - LOG(ERROR, - "Failed to forward non-blocking rpc request reduced remove requests"); - return EBUSY; } } else { // "Big" files for(const auto& endp : CTX->hosts()) { @@ -260,12 +278,14 @@ forward_remove(const std::string& path) { * during a truncate() call. * @param path * @param length + * @param copy Target replica (0 original) * @return error code */ int -forward_decr_size(const std::string& path, size_t length) { +forward_decr_size(const std::string& path, size_t length, const int copy) { - auto endp = CTX->hosts().at(CTX->distributor()->locate_file_metadata(path)); + auto endp = CTX->hosts().at( + CTX->distributor()->locate_file_metadata(path, copy)); try { LOG(DEBUG, "Sending RPC ..."); @@ -295,14 +315,17 @@ forward_decr_size(const std::string& path, size_t length) { * @param path * @param md * @param md_flags + * @param copy Target replica (0 original) * @return error code */ int -forward_update_metadentry( - const string& path, const gkfs::metadata::Metadata& md, - const gkfs::metadata::MetadentryUpdateFlags& md_flags) { +forward_update_metadentry(const string& path, + const gkfs::metadata::Metadata& md, + const gkfs::metadata::MetadentryUpdateFlags& md_flags, + const int copy) { - auto endp = CTX->hosts().at(CTX->distributor()->locate_file_metadata(path)); + auto endp = CTX->hosts().at( + CTX->distributor()->locate_file_metadata(path, copy)); try { LOG(DEBUG, "Sending RPC ..."); @@ -348,6 +371,7 @@ forward_update_metadentry( * This marks that this file doesn't have to be accessed directly * Create a new md with the new name, which should have as value the old name * All operations should check blockcnt and extract a NOTEXISTS + * The operations does not support replication * @param oldpath * @param newpath * @param md @@ -358,8 +382,8 @@ int forward_rename(const string& oldpath, const string& newpath, const gkfs::metadata::Metadata& md) { - auto endp = - CTX->hosts().at(CTX->distributor()->locate_file_metadata(oldpath)); + auto endp = CTX->hosts().at( + CTX->distributor()->locate_file_metadata(oldpath, 0)); try { LOG(DEBUG, "Sending RPC ..."); @@ -405,8 +429,8 @@ forward_rename(const string& oldpath, const string& newpath, // TODO(amiranda): hermes will eventually provide a post(endpoint) // returning one result and a broadcast(endpoint_set) returning a // result_set. When that happens we can remove the .at(0) :/ - auto endp2 = - CTX->hosts().at(CTX->distributor()->locate_file_metadata(newpath)); + auto endp2 = CTX->hosts().at( + CTX->distributor()->locate_file_metadata(newpath, 0)); try { LOG(DEBUG, "Sending RPC ..."); @@ -479,53 +503,85 @@ forward_rename(const string& oldpath, const string& newpath, /** * Send an RPC request for an update to the file size. * This is called during a write() call or similar + * A single correct call is needed only to progress. * @param path * @param size * @param offset * @param append_flag + * @param num_copies number of replicas * @return pair */ pair forward_update_metadentry_size(const string& path, const size_t size, - const off64_t offset, const bool append_flag) { + const off64_t offset, const bool append_flag, + const int num_copies) { - auto endp = CTX->hosts().at(CTX->distributor()->locate_file_metadata(path)); - try { - LOG(DEBUG, "Sending RPC ..."); - // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that we - // can retry for RPC_TRIES (see old commits with margo) - // TODO(amiranda): hermes will eventually provide a post(endpoint) - // returning one result and a broadcast(endpoint_set) returning a - // result_set. When that happens we can remove the .at(0) :/ - auto out = ld_network_service - ->post( - endp, path, size, offset, - bool_to_merc_bool(append_flag)) - .get() - .at(0); + std::vector> handles; - LOG(DEBUG, "Got response success: {}", out.err()); + for(auto copy = 0; copy < num_copies + 1; copy++) { + auto endp = CTX->hosts().at( + CTX->distributor()->locate_file_metadata(path, copy)); + try { + LOG(DEBUG, "Sending RPC ..."); + // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that + // we can retry for RPC_TRIES (see old commits with margo) + // TODO(amiranda): hermes will eventually provide a post(endpoint) + // returning one result and a broadcast(endpoint_set) returning a + // result_set. When that happens we can remove the .at(0) :/ + handles.emplace_back( + ld_network_service->post( + endp, path, size, offset, + bool_to_merc_bool(append_flag))); + } catch(const std::exception& ex) { + LOG(ERROR, "while getting rpc output"); + return make_pair(EBUSY, 0); + } + } + auto err = 0; + ssize_t out_size = 0; + auto idx = 0; + bool valid = false; + for(const auto& h : handles) { + try { + // XXX We might need a timeout here to not wait forever for an + // output that never comes? + auto out = h.get().at(0); - if(out.err()) - return make_pair(out.err(), 0); - else - return make_pair(0, out.ret_size()); - } catch(const std::exception& ex) { - LOG(ERROR, "while getting rpc output"); - return make_pair(EBUSY, 0); + if(out.err() != 0) { + LOG(ERROR, "Daemon {} reported error: {}", idx, out.err()); + } else { + valid = true; + out_size = out.ret_size(); + } + + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to get rpc output"); + if(!valid) { + err = EIO; + } + } + idx++; } + + if(!valid) + return make_pair(err, 0); + else + return make_pair(0, out_size); } + /** * Send an RPC request to get the current file size. * This is called during a lseek() call * @param path + * @param copy Target replica (0 original) * @return pair */ pair -forward_get_metadentry_size(const std::string& path) { +forward_get_metadentry_size(const std::string& path, const int copy) { - auto endp = CTX->hosts().at(CTX->distributor()->locate_file_metadata(path)); + auto endp = CTX->hosts().at( + CTX->distributor()->locate_file_metadata(path, copy)); try { LOG(DEBUG, "Sending RPC ..."); @@ -831,7 +887,8 @@ forward_get_dirents_single(const string& path, int server) { int forward_mk_symlink(const std::string& path, const std::string& target_path) { - auto endp = CTX->hosts().at(CTX->distributor()->locate_file_metadata(path)); + auto endp = + CTX->hosts().at(CTX->distributor()->locate_file_metadata(path, 0)); try { LOG(DEBUG, "Sending RPC ..."); diff --git a/src/common/rpc/distributor.cpp b/src/common/rpc/distributor.cpp index e222a4aa31e45c443c7199ce61b495764ddc4d62..e0f39a9bc65244b1530ef753bf191c33d076f043 100644 --- a/src/common/rpc/distributor.cpp +++ b/src/common/rpc/distributor.cpp @@ -47,27 +47,34 @@ SimpleHashDistributor::localhost() const { return localhost_; } +unsigned int +SimpleHashDistributor::hosts_size() const { + return hosts_size_; +} + host_t -SimpleHashDistributor::locate_data(const string& path, - const chunkid_t& chnk_id) const { - return str_hash(path + ::to_string(chnk_id)) % hosts_size_; +SimpleHashDistributor::locate_data(const string& path, const chunkid_t& chnk_id, + const int num_copy) const { + return (str_hash(path + ::to_string(chnk_id)) + num_copy) % hosts_size_; } host_t SimpleHashDistributor::locate_data(const string& path, const chunkid_t& chnk_id, - unsigned int hosts_size) { + unsigned int hosts_size, + const int num_copy) { if(hosts_size_ != hosts_size) { hosts_size_ = hosts_size; all_hosts_ = std::vector(hosts_size); ::iota(all_hosts_.begin(), all_hosts_.end(), 0); } - return str_hash(path + ::to_string(chnk_id)) % hosts_size_; + return (str_hash(path + ::to_string(chnk_id)) + num_copy) % hosts_size_; } host_t -SimpleHashDistributor::locate_file_metadata(const string& path) const { - return str_hash(path) % hosts_size_; +SimpleHashDistributor::locate_file_metadata(const string& path, + const int num_copy) const { + return (str_hash(path) + num_copy) % hosts_size_; } ::vector @@ -75,6 +82,61 @@ SimpleHashDistributor::locate_directory_metadata(const string& path) const { return all_hosts_; } + +ECCDistributor::ECCDistributor(host_t localhost, unsigned int hosts_size, + unsigned int ecc_size) + : localhost_(localhost), hosts_size_(hosts_size), ecc_size_(ecc_size), + all_hosts_(hosts_size) { + ::iota(all_hosts_.begin(), all_hosts_.end(), 0); +} + +ECCDistributor::ECCDistributor() {} + +host_t +ECCDistributor::localhost() const { + return localhost_; +} + +unsigned int +ECCDistributor::hosts_size() const { + return hosts_size_; +} + +host_t +ECCDistributor::locate_data(const string& path, const chunkid_t& chnk_id, + const int num_copy) const { + if(num_copy == 0) { + return (str_hash(path + ::to_string(chnk_id))) % + (hosts_size_ - ecc_size_); + } else { + return (num_copy + (hosts_size_ - ecc_size_)); + } +} + +host_t +ECCDistributor::locate_data(const string& path, const chunkid_t& chnk_id, + unsigned int hosts_size, const int num_copy) { + if(hosts_size_ != hosts_size) { + hosts_size_ = hosts_size; + all_hosts_ = std::vector(hosts_size); + ::iota(all_hosts_.begin(), all_hosts_.end(), 0); + } + + return (locate_data(path, chnk_id, num_copy)); +} + +host_t +ECCDistributor::locate_file_metadata(const string& path, + const int num_copy) const { + return (str_hash(path) + num_copy) % hosts_size_; +} + +::vector +ECCDistributor::locate_directory_metadata(const string& path) const { + return all_hosts_; +} + + LocalOnlyDistributor::LocalOnlyDistributor(host_t localhost) : localhost_(localhost) {} @@ -83,14 +145,20 @@ LocalOnlyDistributor::localhost() const { return localhost_; } +unsigned int +LocalOnlyDistributor::hosts_size() const { + return hosts_size_; +} + host_t -LocalOnlyDistributor::locate_data(const string& path, - const chunkid_t& chnk_id) const { +LocalOnlyDistributor::locate_data(const string& path, const chunkid_t& chnk_id, + const int num_copy) const { return localhost_; } host_t -LocalOnlyDistributor::locate_file_metadata(const string& path) const { +LocalOnlyDistributor::locate_file_metadata(const string& path, + const int num_copy) const { return localhost_; } @@ -110,24 +178,32 @@ ForwarderDistributor::localhost() const { return fwd_host_; } +unsigned int +ForwarderDistributor::hosts_size() const { + return hosts_size_; +} + host_t ForwarderDistributor::locate_data(const std::string& path, - const chunkid_t& chnk_id) const { + const chunkid_t& chnk_id, + const int num_copy) const { return fwd_host_; } host_t ForwarderDistributor::locate_data(const std::string& path, const chunkid_t& chnk_id, - unsigned int host_size) { + unsigned int host_size, const int num_copy) { return fwd_host_; } host_t -ForwarderDistributor::locate_file_metadata(const std::string& path) const { - return str_hash(path) % hosts_size_; +ForwarderDistributor::locate_file_metadata(const std::string& path, + const int num_copy) const { + return (str_hash(path) + num_copy) % hosts_size_; } + std::vector ForwarderDistributor::locate_directory_metadata(const std::string& path) const { return all_hosts_; @@ -213,21 +289,26 @@ GuidedDistributor::localhost() const { return localhost_; } +unsigned int +GuidedDistributor::hosts_size() const { + return hosts_size_; +} + host_t GuidedDistributor::locate_data(const string& path, const chunkid_t& chnk_id, - unsigned int hosts_size) { + unsigned int hosts_size, const int num_copy) { if(hosts_size_ != hosts_size) { hosts_size_ = hosts_size; all_hosts_ = std::vector(hosts_size); ::iota(all_hosts_.begin(), all_hosts_.end(), 0); } - return (locate_data(path, chnk_id)); + return (locate_data(path, chnk_id, num_copy)); } host_t -GuidedDistributor::locate_data(const string& path, - const chunkid_t& chnk_id) const { +GuidedDistributor::locate_data(const string& path, const chunkid_t& chnk_id, + const int num_copy) const { auto it = map_interval.find(path); if(it != map_interval.end()) { auto it_f = it->second.first.IsInsideInterval(chnk_id); @@ -245,14 +326,16 @@ GuidedDistributor::locate_data(const string& path, } auto locate = path + ::to_string(chnk_id); - return str_hash(locate) % hosts_size_; + return (str_hash(locate) + num_copy) % hosts_size_; } host_t -GuidedDistributor::locate_file_metadata(const string& path) const { - return str_hash(path) % hosts_size_; +GuidedDistributor::locate_file_metadata(const string& path, + const int num_copy) const { + return (str_hash(path) + num_copy) % hosts_size_; } + ::vector GuidedDistributor::locate_directory_metadata(const string& path) const { return all_hosts_; diff --git a/src/common/rpc/rpc_util.cpp b/src/common/rpc/rpc_util.cpp index 46970b82fdc0d793093092c8fc59f34acbbaa3a1..1016ea9ff841eed4e6b62a62c478cc4cae5b9f46 100644 --- a/src/common/rpc/rpc_util.cpp +++ b/src/common/rpc/rpc_util.cpp @@ -36,6 +36,7 @@ extern "C" { #include + using namespace std; namespace gkfs::rpc { @@ -104,4 +105,111 @@ get_host_by_name(const string& hostname) { } #endif +/** + * @brief Get the bit from a bit vector + * + * @param data + * @param position + * @return the bit + */ +bool +get_bitset(const std::vector& data, const uint16_t position) { + return (data[(position) / 8] & 1 << ((position) % 8)); +} + +/** + * @brief Get the bit from a bit vector + * + * @param data + * @param position + */ +void +set_bitset(std::vector& data, const uint16_t position) { + data[(position) / 8] |= 1 << ((position) % 8); // set +} + +std::string +base64_encode(const std::vector& data) { + static const std::string base64_chars = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + std::ostringstream encoded; + uint32_t buffer = 0; + int padding = 0; + + for(uint8_t byte : data) { + buffer = (buffer << 8) | byte; + padding += 8; + while(padding >= 6) { + padding -= 6; + encoded << base64_chars[(buffer >> padding) & 0x3F]; + } + } + + if(padding > 0) { + buffer <<= 6 - padding; + encoded << base64_chars[buffer & 0x3F]; + } + + while(encoded.str().length() % 4 != 0) { + encoded << '='; + } + + return encoded.str(); +} + +std::vector +base64_decode(const std::string& encoded) { + static const std::string base64_chars = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + std::vector data; + uint32_t buffer = 0; + int padding = 0; + size_t count = 0; + + for(char c : encoded) { + if(c == '=') + break; + + std::size_t value = base64_chars.find(c); + if(value == std::string::npos) + continue; + + buffer = (buffer << 6) | value; + padding += 6; + + if(padding >= 8) { + padding -= 8; + data.push_back(static_cast((buffer >> padding) & 0xFF)); + count++; + } + } + + // Handle padding characters + if(padding > 0 && padding < 6 && (buffer & ((1 << padding) - 1)) == 0) { + // Remove the padding bits + buffer >>= padding; + padding = 0; + data.push_back(static_cast((buffer >> 8) & 0xFF)); + count++; + } + + if(count == 0 || padding % 8 != 0) + return {}; + + return data; +} + +std::string +compressBitset(const std::vector& bytes) { + return base64_encode(bytes); +} + +std::vector +decompressBitset(const std::string& compressedString) { + return base64_decode(compressedString); +} + + } // namespace gkfs::rpc \ No newline at end of file diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index 35f636a532a61f9f18c6f5e18816ca21323141ce..46e87a2004e05f9b18a2be287e10c3985cd95385 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -40,6 +40,7 @@ #include #include +#include #include #include #include @@ -51,9 +52,9 @@ #define AGIOS_WRITE 1 #define AGIOS_SERVER_ID_IGNORE 0 #endif - using namespace std; + namespace { /** @@ -115,6 +116,8 @@ rpc_srv_write(hg_handle_t handle) { __func__, in.path, in.chunk_start, in.chunk_end, in.chunk_n, in.total_chunk_size, bulk_size, in.offset); + std::vector write_ops_vect = + gkfs::rpc::decompressBitset(in.wbitset); #ifdef GKFS_ENABLE_AGIOS int* data; @@ -229,8 +232,9 @@ rpc_srv_write(hg_handle_t handle) { chnk_id_file++) { // Continue if chunk does not hash to this host #ifndef GKFS_ENABLE_FORWARDING - if(RPC_DATA->distributor()->locate_data(in.path, chnk_id_file, - host_size) != host_id) { + + if(!(gkfs::rpc::get_bitset(write_ops_vect, + chnk_id_file - in.chunk_start))) { GKFS_DATA->spdlogger()->trace( "{}() chunkid '{}' ignored as it does not match to this host with id '{}'. chnk_id_curr '{}'", __func__, chnk_id_file, host_id, chnk_id_curr); @@ -240,8 +244,10 @@ rpc_srv_write(hg_handle_t handle) { if(GKFS_DATA->enable_chunkstats()) { GKFS_DATA->stats()->add_write(in.path, chnk_id_file); } -#endif +#endif + GKFS_DATA->spdlogger()->error("{}() Processing at host {} -> {}", + __func__, host_id, chnk_id_file); chnk_ids_host[chnk_id_curr] = chnk_id_file; // save this id to host chunk list // offset case. Only relevant in the first iteration of the loop and if @@ -417,7 +423,8 @@ rpc_srv_read(hg_handle_t handle) { "{}() path: '{}' chunk_start '{}' chunk_end '{}' chunk_n '{}' total_chunk_size '{}' bulk_size: '{}' offset: '{}'", __func__, in.path, in.chunk_start, in.chunk_end, in.chunk_n, in.total_chunk_size, bulk_size, in.offset); - + std::vector read_bitset_vect = + gkfs::rpc::decompressBitset(in.wbitset); #ifdef GKFS_ENABLE_AGIOS int* data; ABT_eventual eventual = ABT_EVENTUAL_NULL; @@ -520,8 +527,10 @@ rpc_srv_read(hg_handle_t handle) { chnk_id_file++) { // Continue if chunk does not hash to this host #ifndef GKFS_ENABLE_FORWARDING - if(RPC_DATA->distributor()->locate_data(in.path, chnk_id_file, - host_size) != host_id) { + // We only check if we are not using replicas + + if(!(gkfs::rpc::get_bitset(read_bitset_vect, + chnk_id_file - in.chunk_start))) { GKFS_DATA->spdlogger()->trace( "{}() chunkid '{}' ignored as it does not match to this host with id '{}'. chnk_id_curr '{}'", __func__, chnk_id_file, host_id, chnk_id_curr); @@ -530,6 +539,7 @@ rpc_srv_read(hg_handle_t handle) { if(GKFS_DATA->enable_chunkstats()) { GKFS_DATA->stats()->add_read(in.path, chnk_id_file); } + #endif chnk_ids_host[chnk_id_curr] = @@ -597,6 +607,10 @@ rpc_srv_read(hg_handle_t handle) { GKFS_DATA->spdlogger()->warn( "{}() Not all chunks were detected!!! Size left {}", __func__, chnk_size_left_host); + + if(chnk_size_left_host == in.total_chunk_size) + return HG_CANCELED; + /* * 4. Read task results and accumulate in out.io_size */ diff --git a/tests/scripts/compile_dep.sh/0.9.3-exp/agios.out b/tests/scripts/compile_dep.sh/0.9.3-exp/agios.out new file mode 100644 index 0000000000000000000000000000000000000000..b098aecfac7bbfa0a0ce7623d4bd3895af481285 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3-exp/agios.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### + diff --git a/tests/scripts/compile_dep.sh/0.9.3-exp/all.out b/tests/scripts/compile_dep.sh/0.9.3-exp/all.out new file mode 100644 index 0000000000000000000000000000000000000000..4e2d6199336134da0058f72a0492a03e8d9ac7d3 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3-exp/all.out @@ -0,0 +1,17 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### +######## Installing: gf-complete ############################### +######## Installing: Jerasure ############################### +Done \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.3-exp/ci.out b/tests/scripts/compile_dep.sh/0.9.3-exp/ci.out new file mode 100644 index 0000000000000000000000000000000000000000..d3c276800bec7125a9c3247dc2b2218fe77fb5f2 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3-exp/ci.out @@ -0,0 +1,9 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3-exp/default.out b/tests/scripts/compile_dep.sh/0.9.3-exp/default.out new file mode 100644 index 0000000000000000000000000000000000000000..787e4a08f0946c4b6155af6181afabab9842ea28 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3-exp/default.out @@ -0,0 +1,10 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3-exp/direct.out b/tests/scripts/compile_dep.sh/0.9.3-exp/direct.out new file mode 100644 index 0000000000000000000000000000000000000000..816266773b569d3d098245c7b868f7983753ff12 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3-exp/direct.out @@ -0,0 +1,6 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3-exp/infiniband_verbs.out b/tests/scripts/compile_dep.sh/0.9.3-exp/infiniband_verbs.out new file mode 100644 index 0000000000000000000000000000000000000000..e82dd020fa7dabd6cbd95dae0a07f38e2fabb825 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3-exp/infiniband_verbs.out @@ -0,0 +1,10 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric%verbs ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3-exp/marenostrum4.out b/tests/scripts/compile_dep.sh/0.9.3-exp/marenostrum4.out new file mode 100644 index 0000000000000000000000000000000000000000..38f67ce3df9c4cbeee8d31ce3e1da6725d8f5819 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3-exp/marenostrum4.out @@ -0,0 +1,13 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3-exp/mogon2.out b/tests/scripts/compile_dep.sh/0.9.3-exp/mogon2.out new file mode 100644 index 0000000000000000000000000000000000000000..38f67ce3df9c4cbeee8d31ce3e1da6725d8f5819 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3-exp/mogon2.out @@ -0,0 +1,13 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3-exp/ngio.out b/tests/scripts/compile_dep.sh/0.9.3-exp/ngio.out new file mode 100644 index 0000000000000000000000000000000000000000..38f67ce3df9c4cbeee8d31ce3e1da6725d8f5819 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3-exp/ngio.out @@ -0,0 +1,13 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3-exp/omnipath_psm2.out b/tests/scripts/compile_dep.sh/0.9.3-exp/omnipath_psm2.out new file mode 100644 index 0000000000000000000000000000000000000000..7138da9e21f4b03645a9cfe0ff07bb66276ca00a --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3-exp/omnipath_psm2.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3-exp/p9.out b/tests/scripts/compile_dep.sh/0.9.3-exp/p9.out new file mode 100644 index 0000000000000000000000000000000000000000..7138da9e21f4b03645a9cfe0ff07bb66276ca00a --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3-exp/p9.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/dl_dep.sh/0.9.3-exp/agios.out b/tests/scripts/dl_dep.sh/0.9.3-exp/agios.out new file mode 100644 index 0000000000000000000000000000000000000000..bf5455ca27763dc18c29eaf971987d46b9a61e7f --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3-exp/agios.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3-exp/all.out b/tests/scripts/dl_dep.sh/0.9.3-exp/all.out new file mode 100644 index 0000000000000000000000000000000000000000..726ea69af840e92ba6ec01db8b07606fa6505b0b --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3-exp/all.out @@ -0,0 +1,16 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags '' +Cloned 'https://github.com/ceph/gf-complete' to 'gf-complete' with commit '[a6862d10c9db467148f20eef2c6445ac9afd94d8]' +Cloned 'https://github.com/tsuraan/Jerasure' to 'Jerasure' with commit '[414c96ef2b9934953b6facb31d803d79b1dd1405]' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3-exp/ci.out b/tests/scripts/dl_dep.sh/0.9.3-exp/ci.out new file mode 100644 index 0000000000000000000000000000000000000000..4ee089ef7a35040859946174aa99c82d790e6049 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3-exp/ci.out @@ -0,0 +1,10 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3-exp/default.out b/tests/scripts/dl_dep.sh/0.9.3-exp/default.out new file mode 100644 index 0000000000000000000000000000000000000000..a68701cf0564db9c2cf70246d63f8da9ffdc4f65 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3-exp/default.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3-exp/direct.out b/tests/scripts/dl_dep.sh/0.9.3-exp/direct.out new file mode 100644 index 0000000000000000000000000000000000000000..e27eae530550700bd1de2bfa29823180c8ca551f --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3-exp/direct.out @@ -0,0 +1,7 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3-exp/infiniband_verbs.out b/tests/scripts/dl_dep.sh/0.9.3-exp/infiniband_verbs.out new file mode 100644 index 0000000000000000000000000000000000000000..6ebc289ff6e5a1ac087078ee33ca587d1a9a8627 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3-exp/infiniband_verbs.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3-exp/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.3-exp/marenostrum4.out new file mode 100644 index 0000000000000000000000000000000000000000..657154fdc1ec4e756f29281558459fa9662bd83d --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3-exp/marenostrum4.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3-exp/mogon2.out b/tests/scripts/dl_dep.sh/0.9.3-exp/mogon2.out new file mode 100644 index 0000000000000000000000000000000000000000..657154fdc1ec4e756f29281558459fa9662bd83d --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3-exp/mogon2.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3-exp/ngio.out b/tests/scripts/dl_dep.sh/0.9.3-exp/ngio.out new file mode 100644 index 0000000000000000000000000000000000000000..657154fdc1ec4e756f29281558459fa9662bd83d --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3-exp/ngio.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3-exp/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.3-exp/omnipath_psm2.out new file mode 100644 index 0000000000000000000000000000000000000000..d26b498ca41977f2e48d86f1f0437c1bc0b7a563 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3-exp/omnipath_psm2.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3-exp/p9.out b/tests/scripts/dl_dep.sh/0.9.3-exp/p9.out new file mode 100644 index 0000000000000000000000000000000000000000..3a99bb523089fa9d7aaa1a36a4a62fa9d6295732 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3-exp/p9.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Done diff --git a/tests/unit/test_guided_distributor.cpp b/tests/unit/test_guided_distributor.cpp index eb36647afb729f195a92183bf53cf3fbf69fb8ff..36b6f59103dc07a9995c7ecd19655b00802db4aa 100644 --- a/tests/unit/test_guided_distributor.cpp +++ b/tests/unit/test_guided_distributor.cpp @@ -39,23 +39,23 @@ TEST_CASE( "Guided distributor Testing", "[Distributor]" ) { // The distributor should return 3 for all the tested files auto d = gkfs::rpc::GuidedDistributor(); - REQUIRE( d.locate_data("/t.c01",1,10) == 3 ); - REQUIRE( d.locate_data("/t.c02",1,10) == 3 ); - REQUIRE( d.locate_data("/t.c03",1,10) == 3 ); - REQUIRE( d.locate_data("/t.c04",1,10) == 3 ); - REQUIRE( d.locate_data("/t.c05",1,10) == 3 ); - REQUIRE( d.locate_data("/t.c06",1,10) == 3 ); - REQUIRE( d.locate_data("/t.c07",1,10) == 3 ); + REQUIRE( d.locate_data("/t.c01",1,10,0) == 3 ); + REQUIRE( d.locate_data("/t.c02",1,10,0) == 3 ); + REQUIRE( d.locate_data("/t.c03",1,10,0) == 3 ); + REQUIRE( d.locate_data("/t.c04",1,10,0) == 3 ); + REQUIRE( d.locate_data("/t.c05",1,10,0) == 3 ); + REQUIRE( d.locate_data("/t.c06",1,10,0) == 3 ); + REQUIRE( d.locate_data("/t.c07",1,10,0) == 3 ); // Next result is random, but with the same seed is consistent // We ask for chunk 5 that is distributed randomly between the // 10 servers. - REQUIRE ( (d.locate_data("/t.c01",5,10) + - d.locate_data("/t.c02",5,10) + - d.locate_data("/t.c03",5,10) + - d.locate_data("/t.c04",5,10) + - d.locate_data("/t.c05",5,10) + - d.locate_data("/t.c06",5,10) + - d.locate_data("/t.c07",5,10) ) == 42); + REQUIRE ( (d.locate_data("/t.c01",5,10,0) + + d.locate_data("/t.c02",5,10,0) + + d.locate_data("/t.c03",5,10,0) + + d.locate_data("/t.c04",5,10,0) + + d.locate_data("/t.c05",5,10,0) + + d.locate_data("/t.c06",5,10,0) + + d.locate_data("/t.c07",5,10,0) ) == 42); } }