From 23250a96caf83f76b968b6518182ef3e69e35f31 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 11:30:48 -0300 Subject: [PATCH 01/71] Create a forwarding mode in GekkoFS --- .gitlab-ci.yml | 155 +++++++++++++++++++++++++---- CMake/FindAGIOS.cmake | 18 ++++ CMakeLists.txt | 12 +++ agios/access_times.func | 11 ++ agios/agios.conf | 92 +++++++++++++++++ docker/build_env.docker | 5 + docker/debian_build_env.docker | 2 + include/client/env.hpp | 3 + include/client/gkfs_functions.hpp | 3 + include/client/hooks.hpp | 5 + include/client/preload_context.hpp | 5 + include/client/preload_util.hpp | 1 + include/config.hpp | 1 + include/daemon/scheduler/agios.hpp | 15 +++ include/global/rpc/distributor.hpp | 17 ++++ scripts/compile_dep.sh | 20 +++- scripts/dl_dep.sh | 13 ++- src/client/gkfs_functions.cpp | 43 ++++++++ src/client/hooks.cpp | 28 +++++- src/client/intercept.cpp | 16 ++- src/client/preload.cpp | 47 +++++++++ src/client/preload_context.cpp | 9 +- src/client/preload_util.cpp | 68 ++++++++++++- src/daemon/CMakeLists.txt | 4 + src/daemon/daemon.cpp | 19 ++++ src/daemon/handler/srv_data.cpp | 82 +++++++++++++++ src/daemon/scheduler/agios.cpp | 10 ++ src/global/rpc/distributor.cpp | 27 ++++- 28 files changed, 699 insertions(+), 32 deletions(-) create mode 100644 CMake/FindAGIOS.cmake create mode 100644 agios/access_times.func create mode 100644 agios/agios.conf create mode 100644 include/daemon/scheduler/agios.hpp create mode 100644 src/daemon/scheduler/agios.cpp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1a2457ca3..9b1ac1266 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,28 +2,44 @@ stages: - download deps - build deps - build + - build forwarding - test + - test forwarding variables: - DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" - DEPS_INSTALL_PATH: "${CI_PROJECT_DIR}/deps/install" - DEPS_COMMIT: "${CI_PROJECT_DIR}/deps/install/gkfs_deps_commit" - BUILD_PATH: "${CI_PROJECT_DIR}/gkfs/build" - INSTALL_PATH: "${CI_PROJECT_DIR}/gkfs/install" - INTEGRATION_TESTS_BIN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration" - INTEGRATION_TESTS_RUN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/run" - TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build" - PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" - LOG_PATH: "${CI_PROJECT_DIR}/logs" - LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/deps/install/lib;${CI_PROJECT_DIR}/deps/install/lib64" + DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" + DEPS_INSTALL_PATH: "${CI_PROJECT_DIR}/deps/install" + DEPS_COMMIT: "${CI_PROJECT_DIR}/deps/install/gkfs_deps_commit" + + BUILD_PATH: "${CI_PROJECT_DIR}/gkfs/build" + INSTALL_PATH: "${CI_PROJECT_DIR}/gkfs/install" + INTEGRATION_TESTS_BIN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration" + INTEGRATION_TESTS_RUN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/run" + + BUILD_FWD_PATH: "${CI_PROJECT_DIR}/gkfwd/build" + INSTALL_FWD_PATH: "${CI_PROJECT_DIR}/gkfwd/install" + INTEGRATION_TESTS_FWD_BIN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration" + INTEGRATION_TESTS_FWD_RUN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/run" + + TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/gkfs/build" + PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" + + TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test/gkfwd/build" + PYTEST_FWD: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" + + LOG_PATH: "${CI_PROJECT_DIR}/logs" + + LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/deps/install/lib;${CI_PROJECT_DIR}/deps/install/lib64" # Configuration variables - GKFS_LOG_LEVEL: "100" - GKFS_DAEMON_LOG_PATH: "${CI_PROJECT_DIR}/logs/daemon.log" - LIBGKFS_LOG: "all" - LIBGKFS_LOG_OUTPUT: "${CI_PROJECT_DIR}/logs/gkfs_client.log" - GIT_SUBMODULE_STRATEGY: recursive + GKFS_LOG_LEVEL: "100" + GKFS_DAEMON_LOG_PATH: "${CI_PROJECT_DIR}/logs/daemon.log" + LIBGKFS_LOG: "all" + LIBGKFS_LOG_OUTPUT: "${CI_PROJECT_DIR}/logs/gkfs_client.log" + GIT_SUBMODULE_STRATEGY: recursive -image: gekkofs/gekkofs:build_env-0.8.0 +# Temporary new image file +# image: gekkofs/gekkofs:build_env-0.8.0 +image: jeanbez/gekkofs-forwarding:latest compile dependencies: stage: build deps @@ -65,6 +81,29 @@ compile GekkoFS: paths: - ${INSTALL_PATH} +compile GekkoFWD: + stage: build + dependencies: + - "compile dependencies" + script: + - mkdir -p ${BUILD_FWD_PATH} && cd ${BUILD_FWD_PATH} + - cmake + -Wdev + -Wdeprecate + -DCMAKE_BUILD_TYPE=Debug + -DGKFS_BUILD_TESTS:BOOL=ON + -DGKFS_INSTALL_TESTS:BOOL=ON + -DRPC_PROTOCOL="ofi+sockets" + -DENABLE_FORWARDING:BOOL=ON \ + -DENABLE_AGIOS:BOOL=ON \ + -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} + -DCMAKE_INSTALL_PREFIX=${INSTALL_FWD_PATH} + ${CI_PROJECT_DIR} + - make -j$(nproc) install + artifacts: + paths: + - ${INSTALL_FWD_PATH} + compile tests: stage: build dependencies: @@ -147,3 +186,85 @@ test lseek: when: on_failure paths: - "${LOG_PATH}" + +test wr: + stage: test forwarding + dependencies: + - "compile GekkoFWD" + script: + - mkdir -p "${LOG_PATH}" + - cp agios/* /tmp + - echo "`hostname` 0" > gkfs_forwarding.map + - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & + - sleep 4 + - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_wr + artifacts: + when: on_failure + paths: + - "${LOG_PATH}" + +test directories: + stage: test forwarding + dependencies: + - "compile GekkoFWD" + script: + - mkdir -p "${LOG_PATH}" + - cp agios/* /tmp + - echo "`hostname` 0" > gkfs_forwarding.map + - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & + - sleep 4 + - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_dir + artifacts: + when: on_failure + paths: + - "${LOG_PATH}" + +test truncate: + stage: test forwarding + dependencies: + - "compile GekkoFWD" + script: + - mkdir -p "${LOG_PATH}" + - cp agios/* /tmp + - echo "`hostname` 0" > gkfs_forwarding.map + - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & + - sleep 4 + - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_truncate + artifacts: + when: on_failure + paths: + - "${LOG_PATH}" + +test path resolution: + stage: test forwarding + dependencies: + - "compile GekkoFWD" + script: + - mkdir -p "${LOG_PATH}" + - cp agios/* /tmp + - echo "`hostname` 0" > gkfs_forwarding.map + - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & + - sleep 4 + - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_path_resolution + artifacts: + when: on_failure + paths: + - "${LOG_PATH}" + +test lseek: + stage: test forwarding + dependencies: + - "compile GekkoFWD" + script: + - mkdir -p "${LOG_PATH}" + - cp agios/* /tmp + - echo "`hostname` 0" > gkfs_forwarding.map + - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & + - sleep 4 + - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_lseek + artifacts: + when: on_failure + paths: + - "${LOG_PATH}" + + diff --git a/CMake/FindAGIOS.cmake b/CMake/FindAGIOS.cmake new file mode 100644 index 000000000..a11910d33 --- /dev/null +++ b/CMake/FindAGIOS.cmake @@ -0,0 +1,18 @@ +find_path(AGIOS_INCLUDE_DIR + NAMES agios.h +) + +find_library(AGIOS_LIBRARY + NAMES agios +) + +set(AGIOS_INCLUDE_DIRS ${AGIOS_INCLUDE_DIR}) +set(AGIOS_LIBRARIES ${AGIOS_LIBRARY}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(AGIOS DEFAULT_MSG AGIOS_LIBRARIES AGIOS_INCLUDE_DIRS) + +mark_as_advanced( + AGIOS_LIBRARY + AGIOS_INCLUDE_DIR +) \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e6d61290..c515b50bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,7 @@ find_package(Snappy REQUIRED) find_package(ZStd REQUIRED) find_package(JeMalloc) # required if rocksdb has been build with jemalloc find_package(RocksDB REQUIRED) +find_package(AGIOS REQUIRED) # margo dependencies find_package(Mercury REQUIRED) find_package(Abt REQUIRED) @@ -141,6 +142,17 @@ if(ENABLE_CLIENT_LOG) add_definitions(-DGKFS_ENABLE_LOGGING) endif() message(STATUS "[gekkofs] Client logging output: ${ENABLE_CLIENT_LOG}") +option(ENABLE_FORWARDING "Enable forwarding mode" OFF) +if(ENABLE_FORWARDING) + add_definitions(-DGKFS_ENABLE_FORWARDING) +endif() +message(STATUS "[gekkofs] Forwarding mode: ${ENABLE_FORWARDING}") + +option(ENABLE_AGIOS "Enable AGIOS scheduling library" OFF) +if(ENABLE_AGIOS) + add_definitions(-DGKFS_ENABLE_AGIOS) +endif() +message(STATUS "[gekkofs] AGIOS scheduling: ${ENABLE_AGIOS}") set(CLIENT_LOG_MESSAGE_SIZE 1024 CACHE STRING "Maximum size of a log message in the client library") add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${CLIENT_LOG_MESSAGE_SIZE}) diff --git a/agios/access_times.func b/agios/access_times.func new file mode 100644 index 000000000..a6accf355 --- /dev/null +++ b/agios/access_times.func @@ -0,0 +1,11 @@ +2 +8 64 +566.8405 6784.471 +238.2483 1173.456 +145.6056 1018.143 +187.2437 -201.8793 +64 128 +571.2364 -740.0433 +234.6933 41.99119 +218.3765 -6295.179 +113.1743 2235.442 diff --git a/agios/agios.conf b/agios/agios.conf new file mode 100644 index 000000000..e0fc38d0a --- /dev/null +++ b/agios/agios.conf @@ -0,0 +1,92 @@ +library_options: +{ #ATTENTION: with my most recent modification on statistics, the prediction module is NOT WORKING, DO NOT USE IT. + #should we generate trace files during execution? + trace = false ; + + #should we trace predicted requests? (for debug purposes, this trace will not be useful for future executions) + trace_predict = false ; + + #should we make a complete trace, with all scheduler's operations, such as waiting times? (for debug purposes) + trace_full = false ; + + #should the prediction module read trace files? (if false, the Prediction Module is useless) + predict_read_traces = false ; + + #should the prediction module try to predict aggregations? (requires predict_read_traces = true) + predict_request_aggregation = false ; + + #should the prediction module create simplified traces with information (the metrics) it obtained from the real traces? + predict_write_simplified_traces = false; + + #the tolerance for arrival times difference when checking if two predicted requests are the same (in %) + prediction_time_error = 10 + + #this parameter gives the frequency with which the prediction module will redo its predicted aggregations (in number of requests that must be processed between refreshs). This is necessary because these predictions use a factor that represents the ability to overlap waiting times with processing of other requests. At initialization, this factor will be calculated from the provided trace files, but during execution it can be recalculated using measurements for this ability during the actual scheduling. If the parameter is set to -1, aggregations will not be recalculated during execution. (in number of requests) + prediction_recalculate_alpha_period = -1 + + #prefix and sufix for trace files (with path). Their names must be trace_file_prefix+"."+number+"."+trace_file_sufix, with ordered numbers (no holes) + trace_file_prefix = "/tmp/agios_tracefile" + trace_file_sufix = "out" + #prefix for simple trace files (with path). Their names will be prefix+"."+number+"."+trace_file_sufix + simple_trace_prefix = "/tmp/agios_simpletracefile" + + #parameters used by aIOLi and MLF + waiting_time = 900000 + aioli_quantum = 65536 + mlf_quantum = 8192 + + #parameter used by TW (ms) + time_window_size = 1000 #the paper proposing TW recommends 1000 for HDD and 250 for SSD. + + #file (with path) with access times functions (generated by SeRRa - http://serratoool.bitbucket.org/). Used by aIOLi to quantum assignment and by the mechanism that automatically selects the best scheduling algorithm to use. If you are using a static algorithm which is not aIOLi, this does not matter, but you need to provide it anyway. In this case, you can use the one provided as example with the library source code + access_times_func_file = "/tmp/access_times.func" + + #to how many scheduling algorithms the performance module keeps measurements. When we are changing scheduling algorithms, we may observe new measurements (through the agios_release_request function) to the previous algorithms, so we could update information we have for them. It makes no sense to have a big value for performance_values if we don't change algorithms too often + performance_values = 5 + + #how many scheduling algorithms changes we register in the stats file? + proc_algs = 1000 + + #default I/O scheduling algorithm to use (the one to be used if the previous value was set to false) + #existing algorithms (case sensitive): "MLF", "aIOLi", "SJF", "TO", "TO-agg", "SRTF", "TW", "NOOP", "DYN_TREE", "ARMED_BANDIT" (case sensitive) + # SRTF it's experimental and uses information from trace files (don't use it if these are not available) + # NOOP is the NO operation scheduling algorithm, you should not observe performance improvements by using this one + # TW only makes sense if the user is providing AGIOS correct application id for each request. Don't use it otherwise + # DYN_TREE is a dynamic scheduling algorithm which selects the best (among MLF, AIOLI, SFJ, TO, TO-AGG, and NOOP) according to a decision tree. Use it only when using AGIOS to schedule requests to parallel file system' servers. + # ARMED_BANDIT is another dynamic algorithm which periodically selects between MLF, aIOLi, SJF, TO, TO-agg, and NOOP. It keeps performance measurements for these algorithms and gives more probability to the better ones. + default_algorithm = "TO"; + + # TWINS window size, in us + twins_window_duration = 1000; + dynamic_twins = false; + # time between window size adaptation, in ms + dynamic_twins_period = 1000; + + dynamic_twins_clients = 32; + dynamic_twins_processes = 128; + + # Only relevant if default_algorithm is a dynamic one. this parameter gives the frequency with which the automatic scheduling algorithm selection will recalculate the scheduling algorithm. This selection will be done using the access pattern from this period. If -1 is provided, then the selection will be done at the beginning of execution only (using information from traces for DYN_TREE). The next parameter gives the minimum number of requests which need to happen in this period for the selection to be done (otherwise we will wait longer before recalculating). (in msec). Notice that it makes no sense to use ARMED_BANDIT without setting period and min_reqnumber, as it needs to be iteractive. + select_algorithm_period = 1000 + + select_algorithm_min_reqnumber=1 + + #also for dynamic algorithms. says if TW should be one of the options considered by the dynamic scheduler. TW requires identifying requests according to the applications they come from. If this is not possible, don't use TW! + enable_TW = false ; + + #if default_algorithm is a dynamic algorithm, you need to indicate which static algorithm to use at first (before automatically selecting the next one). Only relevant if you are using ARMED_BANDIT + starting_algorithm = "SJF" ; + + # ARMED_BANDIT parameters - only relevant if default_algorithm is ARMED_BANDIT, but need to be provided anyway. min_ab_probability gives the minimum probability given to scheduling algorithms which performed poorly. The algorithm needs to maintain some probability to accomodate changes in the access pattern. validity_window is the period of time (in msec) for which performance measurements are still valid, after this time period we discard them (so we adapt to new situations). performance_window determines how many performance measurements the ARMED_BANDIT algorithms keeps for each scheduling algorithm option (taking the average of them). Keeping a huge window takes more memory. Moreover, it is related with validity_window and select_algorithm_period as if we discard measurements too often, we will never fill the whole window. + min_ab_probability = 3 + validity_window = 360000 + performance_window = 10 +}; +user_info: +{ + #stripe size used by the library's users (in bytes). This is used for detecting the access pattern at a parallel file system server. Useless for other uses. + stripe_size = 32768 ; + + #maximum buffer size used for storing trace parts (in KB). Having a buffer avoids generating requests to the local file system, which interfere in performance. On the other hand, having a large buffer can affect performance and decrease available space for data buffer. + max_trace_buffer_size = 32768 ; + +}; diff --git a/docker/build_env.docker b/docker/build_env.docker index 46094c08e..ef83cbaa9 100644 --- a/docker/build_env.docker +++ b/docker/build_env.docker @@ -25,6 +25,7 @@ RUN yum -y -q update && yum -y -q install \ gcc \ gcc-c++ \ openssl-devel \ + libconfig-devel \ # Mercury dependencies libtool \ libtool-ltdl-devel \ @@ -34,6 +35,7 @@ RUN yum -y -q update && yum -y -q install \ snappy-devel \ libzstd-devel \ lz4-devel \ + bzip2 \ bzip2-devel \ # ada-fs requires C++ 14 devtoolset-8-gcc \ @@ -78,9 +80,11 @@ RUN yum -y -q update && yum -y -q install \ automake \ cmake \ cmake3 \ + sudo \ gcc \ gcc-c++ \ openssl-devel \ + libconfig-devel \ # Mercury dependencies libtool \ libtool-ltdl-devel \ @@ -90,6 +94,7 @@ RUN yum -y -q update && yum -y -q install \ snappy-devel \ libzstd-devel \ lz4-devel \ + bzip2 \ bzip2-devel \ # ada-fs requires C++ 14 devtoolset-8-gcc \ diff --git a/docker/debian_build_env.docker b/docker/debian_build_env.docker index b697b49e9..743e10d22 100644 --- a/docker/debian_build_env.docker +++ b/docker/debian_build_env.docker @@ -19,6 +19,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ gcc \ g++ \ + # AGIOS dependencies + libconfig-dev \ # Mercury dependencies libltdl-dev \ lbzip2 \ diff --git a/include/client/env.hpp b/include/client/env.hpp index e438ab7f4..0555371f7 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -33,6 +33,9 @@ static constexpr auto LOG_OUTPUT = ADD_PREFIX("LOG_OUTPUT"); static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); static constexpr auto CWD = ADD_PREFIX("CWD"); static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); +#ifdef GKFS_ENABLE_FORWARDING +static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE"); +#endif } // namespace env } // namespace gkfs diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 7ae71ce99..674174c1c 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -76,6 +76,9 @@ ssize_t gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset); ssize_t gkfs_read(int fd, void* buf, size_t count); +ssize_t gkfs_readv(int fd, const struct iovec* iov, int iovcnt); + +ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset); int gkfs_opendir(const std::string& path); diff --git a/include/client/hooks.hpp b/include/client/hooks.hpp index a0aacee42..d23127bc4 100644 --- a/include/client/hooks.hpp +++ b/include/client/hooks.hpp @@ -40,6 +40,11 @@ int hook_read(unsigned int fd, void* buf, size_t count); int hook_pread(unsigned int fd, char* buf, size_t count, loff_t pos); +int hook_readv(unsigned long fd, const struct iovec * iov, unsigned long iovcnt); + +int hook_preadv(unsigned long fd, const struct iovec * iov, unsigned long iovcnt, + unsigned long pos_l, unsigned long pos_h); + int hook_write(unsigned int fd, const char* buf, size_t count); int hook_pwrite(unsigned int fd, const char* buf, size_t count, loff_t pos); diff --git a/include/client/preload_context.hpp b/include/client/preload_context.hpp index 0eb43be26..616a92966 100644 --- a/include/client/preload_context.hpp +++ b/include/client/preload_context.hpp @@ -82,6 +82,7 @@ private: std::vector hosts_; uint64_t local_host_id_; + uint64_t fwd_host_id_; bool interception_enabled_; @@ -123,6 +124,10 @@ public: void local_host_id(uint64_t id); + uint64_t fwd_host_id() const; + + void fwd_host_id(uint64_t id); + RelativizeStatus relativize_fd_path(int dirfd, const char* raw_path, std::string& relative_path, diff --git a/include/client/preload_util.hpp b/include/client/preload_util.hpp index 37cf4fc51..f2e000ddd 100644 --- a/include/client/preload_util.hpp +++ b/include/client/preload_util.hpp @@ -62,6 +62,7 @@ int metadata_to_stat(const std::string& path, const gkfs::metadata::Metadata& md std::vector> load_hostfile(const std::string& lfpath); void load_hosts(); +void load_forwarding_map(); } // namespace util } // namespace gkfs diff --git a/include/config.hpp b/include/config.hpp index 982eddcf4..83b5f4bc2 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -24,6 +24,7 @@ namespace gkfs { namespace config { constexpr auto hostfile_path = "./gkfs_hosts.txt"; +constexpr auto forwarding_file_path = "./gkfs_forwarding.map"; namespace io { /* diff --git a/include/daemon/scheduler/agios.hpp b/include/daemon/scheduler/agios.hpp new file mode 100644 index 000000000..9ff1104f1 --- /dev/null +++ b/include/daemon/scheduler/agios.hpp @@ -0,0 +1,15 @@ +#ifndef IFS_SCHEDULER_HPP +#define IFS_SCHEDULER_HPP + +#include + +void agios_initialize(); +void agios_shutdown(); + +void *agios_callback(int64_t request_id); +void *agios_callback_aggregated(int64_t *requests, int32_t total); +void *agios_eventual_callback(int64_t request_id, void *info); + +unsigned long long int generate_unique_id(); + +#endif \ No newline at end of file diff --git a/include/global/rpc/distributor.hpp b/include/global/rpc/distributor.hpp index 2a0b79ff3..e2641d298 100644 --- a/include/global/rpc/distributor.hpp +++ b/include/global/rpc/distributor.hpp @@ -69,6 +69,23 @@ public: std::vector locate_directory_metadata(const std::string& path) const override; }; +class ForwarderDistributor : public Distributor { +private: + host_t fwd_host_; + unsigned int hosts_size_; + std::hash str_hash; +public: + ForwarderDistributor(host_t fwhost, unsigned int hosts_size); + + host_t localhost() const override; + + host_t locate_data(const std::string& path, const chunkid_t& chnk_id) const override; + + host_t locate_file_metadata(const std::string& path) const override; + + std::vector locate_directory_metadata(const std::string& path) const override; +}; + } // namespace rpc } // namespace gkfs diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 1a37a58a7..4332685de 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -13,21 +13,21 @@ VALID_DEP_OPTIONS="mogon2 mogon1 direct all" MOGON1_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" "verbs" + "syscall_intercept" "date" "verbs" "agios" ) MOGON2_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "syscall_intercept" "date" "agios" ) DIRECT_DEPS=( - "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" ) ALL_DEPS=( "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "syscall_intercept" "date" "agios" ) usage_short() { @@ -343,6 +343,16 @@ if check_dependency "ofi" "${DEP_CONFIG[@]}"; then fi fi +# AGIOS +if check_dependency "agios" "${DEP_CONFIG[@]}"; then + echo "############################################################ Installing: AGIOS" + CURR=${SOURCE}/agios + prepare_build_dir "${CURR}" + cd "${CURR}"/build + $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" .. + make install +fi + # Mercury if check_dependency "mercury" "${DEP_CONFIG[@]}"; then @@ -418,7 +428,7 @@ if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then CURR=${SOURCE}/syscall_intercept prepare_build_dir "${CURR}" cd "${CURR}"/build - $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF .. + $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL}" -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF .. make install fi diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index ca859984b..60fe48420 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -13,21 +13,21 @@ VALID_DEP_OPTIONS="mogon2 mogon1 direct all" MOGON1_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi-verbs" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "syscall_intercept" "date" "agios" ) MOGON2_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "syscall_intercept" "date" "agios" ) DIRECT_DEPS=( - "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" ) ALL_DEPS=( "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "syscall_intercept" "date" "agios" ) # Stop all backround jobs on interruption. @@ -360,6 +360,11 @@ if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then clonedeps "syscall_intercept" "https://github.com/pmem/syscall_intercept.git" "cc3412a2ad39f2e26cc307d5b155232811d7408e" "" "syscall_intercept.patch" & fi +# get AGIOS +if check_dependency "agios" "${DEP_CONFIG[@]}"; then +clonedeps "agios" "https://github.com/francielizanon/agios.git" "c26a6544200f823ebb8f890dd94e653d148bf226" "-b development" & +fi + # get date if check_dependency "date" "${DEP_CONFIG[@]}"; then clonedeps "date" "https://github.com/HowardHinnant/date.git" "e7e1482087f58913b80a20b04d5c58d9d6d90155" & diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 029304a77..0e1037a2b 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -485,6 +485,49 @@ ssize_t gkfs_read(int fd, void* buf, size_t count) { return ret; } +ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) { + + auto file = CTX->file_map()->get(fd); + auto pos = offset; // keep truck of current position + ssize_t read = 0; + ssize_t ret; + for (int i = 0; i < iovcnt; ++i) { + auto count = (iov + i)->iov_len; + if (count == 0) { + continue; + } + auto buf = (iov + i)->iov_base; + ret = gkfs_pread(file, reinterpret_cast(buf), count, pos); + if (ret == -1) { + break; + } + read += ret; + pos += ret; + + if (static_cast(ret) < count) { + break; + } + } + + if (read == 0) { + return -1; + } + return read; +} + +ssize_t gkfs_readv(int fd, const struct iovec* iov, int iovcnt) { + + auto gkfs_fd = CTX->file_map()->get(fd); + auto pos = gkfs_fd->pos(); // retrieve the current offset + auto ret = gkfs_preadv(fd, iov, iovcnt, pos); + assert(ret != 0); + if (ret < 0) { + return -1; + } + gkfs_fd->pos(pos + ret); + return ret; +} + ssize_t gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset) { auto gkfs_fd = CTX->file_map()->get(fd); return gkfs_pread(gkfs_fd, reinterpret_cast(buf), count, offset); diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index fa437d1ed..6c2539a76 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -176,6 +176,30 @@ int hook_pread(unsigned int fd, char* buf, size_t count, loff_t pos) { return syscall_no_intercept(SYS_pread64, fd, buf, count, pos); } +int hook_readv(unsigned long fd, const struct iovec* iov, unsigned long iovcnt) { + + LOG(DEBUG, "{}() called with fd: {}, iov: {}, iovcnt: {}", + __func__, fd, fmt::ptr(iov), iovcnt); + + if (CTX->file_map()->exist(fd)) { + return with_errno(gkfs::syscall::gkfs_readv(fd, iov, iovcnt)); + } + return syscall_no_intercept(SYS_readv, fd, iov, iovcnt); +} + +int hook_preadv(unsigned long fd, const struct iovec* iov, unsigned long iovcnt, + unsigned long pos_l, unsigned long pos_h) { + + LOG(DEBUG, "{}() called with fd: {}, iov: {}, iovcnt: {}, " + "pos_l: {}," "pos_h: {}", + __func__, fd, fmt::ptr(iov), iovcnt, pos_l, pos_h); + + if (CTX->file_map()->exist(fd)) { + return with_errno(gkfs::syscall::gkfs_preadv(fd, iov, iovcnt, pos_l)); + } + return syscall_no_intercept(SYS_preadv, fd, iov, iovcnt, pos_l); +} + int hook_write(unsigned int fd, const char* buf, size_t count) { LOG(DEBUG, "{}() called with fd: {}, buf: {}, count {}", @@ -220,7 +244,7 @@ int hook_pwritev(unsigned long fd, const struct iovec* iov, unsigned long iovcnt if (CTX->file_map()->exist(fd)) { return with_errno(gkfs::syscall::gkfs_pwritev(fd, iov, iovcnt, pos_l)); } - return syscall_no_intercept(SYS_pwritev, fd, iov, iovcnt); + return syscall_no_intercept(SYS_pwritev, fd, iov, iovcnt, pos_l); } int hook_unlinkat(int dirfd, const char* cpath, int flags) { @@ -760,4 +784,4 @@ int hook_fstatfs(unsigned int fd, struct statfs* buf) { } } // namespace hook -} // namespace gkfs \ No newline at end of file +} // namespace gkfs diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index 9905a0b5d..70434b4c1 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -496,6 +496,20 @@ int hook(long syscall_number, static_cast(arg3)); break; + case SYS_readv: + *result = gkfs::hook::hook_readv(static_cast(arg0), + reinterpret_cast(arg1), + static_cast(arg2)); + break; + + case SYS_preadv: + *result = gkfs::hook::hook_preadv(static_cast(arg0), + reinterpret_cast(arg1), + static_cast(arg2), + static_cast(arg3), + static_cast(arg4)); + break; + case SYS_pwrite64: *result = gkfs::hook::hook_pwrite(static_cast(arg0), reinterpret_cast(arg1), @@ -914,4 +928,4 @@ void stop_interception() { } } // namespace preload -} // namespace gkfs \ No newline at end of file +} // namespace gkfs diff --git a/src/client/preload.cpp b/src/client/preload.cpp index ad09cff32..e4374db76 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -36,6 +36,8 @@ namespace { // make sure that things are only initialized once pthread_once_t init_env_thread = PTHREAD_ONCE_INIT; +pthread_t mapper; +bool forwarding_running; inline void exit_error_msg(int errcode, const string& msg) { @@ -98,9 +100,22 @@ void init_ld_environment_() { } /* Setup distributor */ + #ifdef GKFS_ENABLE_FORWARDING + try { + gkfs::util::load_forwarding_map(); + + LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + } catch (std::exception& e){ + exit_error_msg(EXIT_FAILURE, fmt::format("Unable set the forwarding host '{}'", e.what())); + } + + auto forwarder_dist = std::make_shared(CTX->fwd_host_id(), CTX->hosts().size()); + CTX->distributor(forwarder_dist); + #else auto simple_hash_dist = std::make_shared(CTX->local_host_id(), CTX->hosts().size()); CTX->distributor(simple_hash_dist); + #endif LOG(INFO, "Retrieving file system configuration..."); @@ -111,6 +126,35 @@ void init_ld_environment_() { LOG(INFO, "Environment initialization successful."); } +#ifdef GKFS_ENABLE_FORWARDING +void *forwarding_mapper(void *p) { + while (forwarding_running) { + try { + gkfs::util::load_forwarding_map(); + + LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + } catch (std::exception& e){ + exit_error_msg(EXIT_FAILURE, fmt::format("Unable set the forwarding host '{}'", e.what())); + } + + // Sleeps for 10 seconds + sleep(10); + } +} +#endif + +void init_forwarding_mapper() { + forwarding_running = true; + + pthread_create(&mapper, NULL, forwarding_mapper, NULL); +} + +void destroy_forwarding_mapper() { + forwarding_running = false; + + pthread_join(mapper, NULL); +} + void log_prog_name() { std::string line; std::ifstream cmdline("/proc/self/cmdline"); @@ -170,6 +214,8 @@ void init_preload() { CTX->unprotect_user_fds(); + init_forwarding_mapper(); + gkfs::preload::start_interception(); } @@ -177,6 +223,7 @@ void init_preload() { * Called last when preload library is used with the LD_PRELOAD environment variable */ void destroy_preload() { + destroy_forwarding_mapper(); CTX->clear_hosts(); LOG(DEBUG, "Peer information deleted"); diff --git a/src/client/preload_context.cpp b/src/client/preload_context.cpp index ce696e74e..9b9f3eca6 100644 --- a/src/client/preload_context.cpp +++ b/src/client/preload_context.cpp @@ -121,6 +121,13 @@ void PreloadContext::local_host_id(uint64_t id) { local_host_id_ = id; } +uint64_t PreloadContext::fwd_host_id() const { + return fwd_host_id_; +} + +void PreloadContext::fwd_host_id(uint64_t id) { + fwd_host_id_ = id; +} RelativizeStatus PreloadContext::relativize_fd_path(int dirfd, const char* raw_path, std::string& relative_path, @@ -360,4 +367,4 @@ PreloadContext::unprotect_user_fds() { } } // namespace preload -} // namespace gkfs \ No newline at end of file +} // namespace gkfs diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index 4db07f20e..16d956e9f 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -176,6 +176,72 @@ vector> load_hostfile(const std::string& lfpath) { return hosts; } +#ifdef GKFS_ENABLE_FORWARDING +map load_forwarding_map_file(const std::string& lfpath) { + + LOG(DEBUG, "Loading forwarding map file file: \"{}\"", lfpath); + + ifstream lf(lfpath); + if (!lf) { + throw runtime_error(fmt::format("Failed to open forwarding map file '{}': {}", + lfpath, strerror(errno))); + } + map forwarding_map; + const regex line_re("^(\\S+)\\s+(\\S+)$", + regex::ECMAScript | regex::optimize); + string line; + string host; + uint64_t forwarder; + std::smatch match; + while (getline(lf, line)) { + if (!regex_match(line, match, line_re)) { + + LOG(ERROR, "Unrecognized line format: [path: '{}', line: '{}']", + lfpath, line); + + throw runtime_error( + fmt::format("unrecognized line format: '{}'", line)); + } + host = match[1]; + forwarder = std::stoi(match[2].str()); + forwarding_map[host] = forwarder; + } + return forwarding_map; +} +#endif + +#ifdef GKFS_ENABLE_FORWARDING +void load_forwarding_map() { + string forwarding_map_file; + + forwarding_map_file = gkfs::env::get_var(gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + + map forwarding_map; + + while (forwarding_map.size() == 0) { + try { + forwarding_map = load_forwarding_map_file(forwarding_map_file); + } catch (const exception& e) { + auto emsg = fmt::format("Failed to load forwarding map file: {}", e.what()); + throw runtime_error(emsg); + } + } + + //if (forwarding_map.size() == 0) { + // throw runtime_error(fmt::format("Forwarding map file is empty: '{}'", forwarding_map_file)); + //} + + auto local_hostname = get_my_hostname(true); + + if (forwarding_map.find(local_hostname) == forwarding_map.end()) { + throw runtime_error(fmt::format("Unable to determine the forwarder for host: '{}'", local_hostname)); + } + LOG(INFO, "Forwarding map loaded for '{}' as '{}'", local_hostname, forwarding_map[local_hostname]); + + CTX->fwd_host_id(forwarding_map[local_hostname]); +} +#endif + void load_hosts() { string hostfile; @@ -239,4 +305,4 @@ void load_hosts() { } } // namespace util -} // namespace gkfs \ No newline at end of file +} // namespace gkfs diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index e4ba89177..4df524b9d 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -12,6 +12,7 @@ set(DAEMON_SRC handler/srv_metadata.cpp handler/srv_data.cpp handler/srv_management.cpp + scheduler/agios.cpp ) set(DAEMON_HEADERS ../../include/config.hpp @@ -23,6 +24,7 @@ set(DAEMON_HEADERS ../../include/global/path_util.hpp ../../include/daemon/daemon.hpp ../../include/daemon/util.hpp + ../../include/daemon/scheduler/agios.hpp ../../include/daemon/ops/metadentry.hpp ../../include/daemon/classes/fs_data.hpp ../../include/daemon/classes/rpc_data.hpp @@ -40,6 +42,7 @@ target_link_libraries(gkfs_daemon env_util spdlog fmt::fmt + ${AGIOS_LIBRARIES} # margo libs ${ABT_LIBRARIES} mercury @@ -53,6 +56,7 @@ target_link_libraries(gkfs_daemon target_include_directories(gkfs_daemon PRIVATE + ${AGIOS_INCLUDE_DIRS} ${ABT_INCLUDE_DIRS} ${MARGO_INCLUDE_DIRS} ) diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 5a709f0e3..187aa2926 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -24,6 +24,9 @@ #include #include #include +#ifdef GKFS_ENABLE_AGIOS +#include +#endif #include #include @@ -209,6 +212,22 @@ void init_environment() { } GKFS_DATA->spdlogger()->info("Startup successful. Daemon is ready."); } +#ifdef GKFS_ENABLE_AGIOS +/** + * Initialize the AGIOS scheduling library + */ +void agios_initialize() { + char configuration[] = "/tmp/agios.conf"; + + if (!agios_init(NULL, NULL, configuration, 0)) { + GKFS_DATA->spdlogger()->error("{}() Failed to initialize AGIOS scheduler: '{}'", __func__, configuration); + + agios_exit(); + + throw; + } +} +#endif /** * Destroys the margo, argobots, and mercury environments diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index 553a9eaa5..2c3226abb 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -21,6 +21,13 @@ #include #include +#ifdef GKFS_ENABLE_AGIOS +#include + +#define AGIOS_READ 0 +#define AGIOS_WRITE 1 +#define AGIOS_SERVER_ID_IGNORE 0 +#endif using namespace std; @@ -142,6 +149,36 @@ static hg_return_t rpc_srv_write(hg_handle_t handle) { auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug("{}() path: {}, size: {}, offset: {}", __func__, in.path, bulk_size, in.offset); + #ifdef GKFS_ENABLE_AGIOS + int *data; + ABT_eventual eventual = ABT_EVENTUAL_NULL; + + /* creating eventual */ + ABT_eventual_create(sizeof(int64_t), &eventual); + + unsigned long long int request_id = generate_unique_id(); + char *agios_path = (char*) in.path; + + // We should call AGIOS before chunking (as that is an internal way to handle the requests) + if (!agios_add_request(agios_path, AGIOS_WRITE, in.offset, in.total_chunk_size, request_id, AGIOS_SERVER_ID_IGNORE, agios_eventual_callback, eventual)) { + GKFS_DATA->spdlogger()->error("{}() Failed to send request to AGIOS", __func__); + } else { + GKFS_DATA->spdlogger()->debug("{}() request {} was sent to AGIOS", __func__, request_id); + } + + /* Block until the eventual is signaled */ + ABT_eventual_wait(eventual, (void **)&data); + + unsigned long long int result = *data; + GKFS_DATA->spdlogger()->debug("{}() request {} was unblocked (offset = {})!", __func__, result, in.offset); + + ABT_eventual_free(&eventual); + + // Let AGIOS knows it can release the request, as it is completed + if (!agios_release_request(agios_path, AGIOS_WRITE, in.total_chunk_size, in.offset)) { + GKFS_DATA->spdlogger()->error("{}() Failed to release request from AGIOS", __func__); + } + #endif /* * 2. Set up buffers for pull bulk transfers */ @@ -199,8 +236,10 @@ static hg_return_t rpc_srv_write(hg_handle_t handle) { // Start to look for a chunk that hashes to this host with the first chunk in the buffer for (auto chnk_id_file = in.chunk_start; chnk_id_file < in.chunk_end || chnk_id_curr < in.chunk_n; chnk_id_file++) { // Continue if chunk does not hash to this host + #ifndef GKFS_ENABLE_FORWARDING if (distributor.locate_data(in.path, chnk_id_file) != host_id) continue; + #endif chnk_ids_host[chnk_id_curr] = chnk_id_file; // save this id to host chunk list // offset case. Only relevant in the first iteration of the loop and if the chunk hashes to this host if (chnk_id_file == in.chunk_start && in.offset > 0) { @@ -347,6 +386,36 @@ static hg_return_t rpc_srv_read(hg_handle_t handle) { auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug("{}() path: {}, size: {}, offset: {}", __func__, in.path, bulk_size, in.offset); + #ifdef GKFS_ENABLE_AGIOS + int *data; + ABT_eventual eventual = ABT_EVENTUAL_NULL; + + /* creating eventual */ + ABT_eventual_create(sizeof(int64_t), &eventual); + + unsigned long long int request_id = generate_unique_id(); + char *agios_path = (char*) in.path; + + // We should call AGIOS before chunking (as that is an internal way to handle the requests) + if (!agios_add_request(agios_path, AGIOS_READ, in.offset, in.total_chunk_size, request_id, AGIOS_SERVER_ID_IGNORE, agios_eventual_callback, eventual)) { + GKFS_DATA->spdlogger()->error("{}() Failed to send request to AGIOS", __func__); + } else { + GKFS_DATA->spdlogger()->debug("{}() request {} was sent to AGIOS", __func__, request_id); + } + + /* block until the eventual is signaled */ + ABT_eventual_wait(eventual, (void **)&data); + + unsigned long long int result = *data; + GKFS_DATA->spdlogger()->debug("{}() request {} was unblocked (offset = {})!", __func__, result, in.offset); + + ABT_eventual_free(&eventual); + + // let AGIOS knows it can release the request, as it is completed + if (!agios_release_request(agios_path, AGIOS_READ, in.total_chunk_size, in.offset)) { + GKFS_DATA->spdlogger()->error("{}() Failed to release request from AGIOS", __func__); + } + #endif /* * 2. Set up buffers for pull bulk transfers @@ -367,9 +436,11 @@ static hg_return_t rpc_srv_read(hg_handle_t handle) { GKFS_DATA->spdlogger()->error("{}() Failed to access allocated buffer from bulk handle", __func__); return gkfs::rpc::cleanup_respond(&handle, &in, &out, &bulk_handle); } + #ifndef GKFS_ENABLE_FORWARDING auto const host_id = in.host_id; auto const host_size = in.host_size; gkfs::rpc::SimpleHashDistributor distributor(host_id, host_size); + #endif auto path = make_shared(in.path); // chnk_ids used by this host @@ -397,8 +468,10 @@ static hg_return_t rpc_srv_read(hg_handle_t handle) { // Start to look for a chunk that hashes to this host with the first chunk in the buffer for (auto chnk_id_file = in.chunk_start; chnk_id_file < in.chunk_end || chnk_id_curr < in.chunk_n; chnk_id_file++) { // Continue if chunk does not hash to this host + #ifndef GKFS_ENABLE_FORWARDING if (distributor.locate_data(in.path, chnk_id_file) != host_id) continue; + #endif chnk_ids_host[chnk_id_curr] = chnk_id_file; // save this id to host chunk list // Only relevant in the first iteration of the loop and if the chunk hashes to this host if (chnk_id_file == in.chunk_start && in.offset > 0) { @@ -570,4 +643,13 @@ static hg_return_t rpc_srv_get_chunk_stat(hg_handle_t handle) { DEFINE_MARGO_RPC_HANDLER(rpc_srv_get_chunk_stat) +#ifdef GKFS_ENABLE_AGIOS +void *agios_eventual_callback(int64_t request_id, void *info) { + GKFS_DATA->spdlogger()->debug("{}() custom callback request {} is ready", __func__, request_id); + + ABT_eventual_set((ABT_eventual) info, &request_id, sizeof(int64_t)); + + return 0; +} +#endif diff --git a/src/daemon/scheduler/agios.cpp b/src/daemon/scheduler/agios.cpp new file mode 100644 index 000000000..2648c1c93 --- /dev/null +++ b/src/daemon/scheduler/agios.cpp @@ -0,0 +1,10 @@ +#include + +unsigned long long int generate_unique_id() { + // Calculates the hash of this request + timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + unsigned long long int id = ts.tv_sec*1000000000L + ts.tv_nsec; + + return id; +} \ No newline at end of file diff --git a/src/global/rpc/distributor.cpp b/src/global/rpc/distributor.cpp index ab0597853..cbb2f428f 100644 --- a/src/global/rpc/distributor.cpp +++ b/src/global/rpc/distributor.cpp @@ -68,5 +68,30 @@ locate_directory_metadata(const string& path) const { return {localhost_}; } +ForwarderDistributor:: +ForwarderDistributor(host_t fwhost, unsigned int hosts_size) : + fwd_host_(fwhost), + hosts_size_(hosts_size) +{} + +host_t ForwarderDistributor:: +localhost() const { + return fwd_host_; +} + +host_t ForwarderDistributor:: +locate_data(const std::string& path, const chunkid_t& chnk_id) const { + return fwd_host_; +} + +host_t ForwarderDistributor:: +locate_file_metadata(const std::string& path) const { + return str_hash(path) % hosts_size_; +} + +std::vector ForwarderDistributor:: +locate_directory_metadata(const std::string& path) const { + return {fwd_host_}; +} } // namespace rpc -} // namespace gkfs \ No newline at end of file +} // namespace gkfs -- GitLab From 5944d630abd5250a68be45c6beee449fdaf9602d Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 11:34:50 -0300 Subject: [PATCH 02/71] Restructure CI for GekkoFWD --- .gitlab-ci.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b1ac1266..42960550e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,9 +2,7 @@ stages: - download deps - build deps - build - - build forwarding - test - - test forwarding variables: DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" @@ -187,8 +185,8 @@ test lseek: paths: - "${LOG_PATH}" -test wr: - stage: test forwarding +test FWD wr: + stage: test dependencies: - "compile GekkoFWD" script: @@ -203,8 +201,8 @@ test wr: paths: - "${LOG_PATH}" -test directories: - stage: test forwarding +test FWD directories: + stage: test dependencies: - "compile GekkoFWD" script: @@ -219,8 +217,8 @@ test directories: paths: - "${LOG_PATH}" -test truncate: - stage: test forwarding +test FWD truncate: + stage: test dependencies: - "compile GekkoFWD" script: @@ -235,8 +233,8 @@ test truncate: paths: - "${LOG_PATH}" -test path resolution: - stage: test forwarding +test FWD path resolution: + stage: test dependencies: - "compile GekkoFWD" script: @@ -251,8 +249,8 @@ test path resolution: paths: - "${LOG_PATH}" -test lseek: - stage: test forwarding +test FWD lseek: + stage: test dependencies: - "compile GekkoFWD" script: -- GitLab From 2961efb89e99ee13bf0732662742417a44bba242 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 13:58:16 -0300 Subject: [PATCH 03/71] Define functions only for the forwarding mode --- src/client/preload.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/client/preload.cpp b/src/client/preload.cpp index e4374db76..14ec65f5c 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -36,8 +36,10 @@ namespace { // make sure that things are only initialized once pthread_once_t init_env_thread = PTHREAD_ONCE_INIT; +#ifdef GKFS_ENABLE_FORWARDING pthread_t mapper; bool forwarding_running; +#endif inline void exit_error_msg(int errcode, const string& msg) { @@ -143,17 +145,21 @@ void *forwarding_mapper(void *p) { } #endif +#ifdef GKFS_ENABLE_FORWARDING void init_forwarding_mapper() { forwarding_running = true; pthread_create(&mapper, NULL, forwarding_mapper, NULL); } +#endif +#ifdef GKFS_ENABLE_FORWARDING void destroy_forwarding_mapper() { forwarding_running = false; pthread_join(mapper, NULL); } +#endif void log_prog_name() { std::string line; @@ -214,7 +220,9 @@ void init_preload() { CTX->unprotect_user_fds(); + #ifdef GKFS_ENABLE_FORWARDING init_forwarding_mapper(); + #endif gkfs::preload::start_interception(); } @@ -223,7 +231,9 @@ void init_preload() { * Called last when preload library is used with the LD_PRELOAD environment variable */ void destroy_preload() { + #ifdef GKFS_ENABLE_FORWARDING destroy_forwarding_mapper(); + #endif CTX->clear_hosts(); LOG(DEBUG, "Peer information deleted"); -- GitLab From fa1039edec452926c0828e2195b7466c8722f491 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 14:12:34 -0300 Subject: [PATCH 04/71] Update .gitlab-ci.yml --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 42960550e..e815f15e7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,10 +19,10 @@ variables: INTEGRATION_TESTS_FWD_BIN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration" INTEGRATION_TESTS_FWD_RUN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/run" - TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/gkfs/build" + TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build" PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" - TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test/gkfwd/build" + TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test/build" PYTEST_FWD: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" LOG_PATH: "${CI_PROJECT_DIR}/logs" @@ -92,8 +92,8 @@ compile GekkoFWD: -DGKFS_BUILD_TESTS:BOOL=ON -DGKFS_INSTALL_TESTS:BOOL=ON -DRPC_PROTOCOL="ofi+sockets" - -DENABLE_FORWARDING:BOOL=ON \ - -DENABLE_AGIOS:BOOL=ON \ + -DENABLE_FORWARDING:BOOL=ON + -DENABLE_AGIOS:BOOL=ON -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} -DCMAKE_INSTALL_PREFIX=${INSTALL_FWD_PATH} ${CI_PROJECT_DIR} -- GitLab From 3f62cf66c60c2b70e427d03a94cf165e90071689 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 14:25:59 -0300 Subject: [PATCH 05/71] Working to fix integration tests --- .gitlab-ci.yml | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e815f15e7..c80560ecc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ stages: - build deps - build - test + - test forwarding variables: DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" @@ -102,7 +103,7 @@ compile GekkoFWD: paths: - ${INSTALL_FWD_PATH} -compile tests: +tests GekkoFS: stage: build dependencies: - "compile dependencies" @@ -114,6 +115,18 @@ compile tests: paths: - ${TESTS_BUILD_PATH} +tests GekkoFWD: + stage: build + dependencies: + - "compile dependencies" + script: + - mkdir -p ${TESTS_BUILD_FWD_PATH} && cd ${TESTS_BUILD_FWD_PATH} + - cmake -DCMAKE_BUILD_TYPE=Debug .. + - make -j$(nproc) + artifacts: + paths: + - ${TESTS_BUILD_FWD_PATH} + integration tests: stage: test script: @@ -186,7 +199,7 @@ test lseek: - "${LOG_PATH}" test FWD wr: - stage: test + stage: test forwarding dependencies: - "compile GekkoFWD" script: @@ -202,7 +215,7 @@ test FWD wr: - "${LOG_PATH}" test FWD directories: - stage: test + stage: test forwarding dependencies: - "compile GekkoFWD" script: @@ -218,7 +231,7 @@ test FWD directories: - "${LOG_PATH}" test FWD truncate: - stage: test + stage: test forwarding dependencies: - "compile GekkoFWD" script: @@ -234,7 +247,7 @@ test FWD truncate: - "${LOG_PATH}" test FWD path resolution: - stage: test + stage: test forwarding dependencies: - "compile GekkoFWD" script: @@ -250,7 +263,7 @@ test FWD path resolution: - "${LOG_PATH}" test FWD lseek: - stage: test + stage: test forwarding dependencies: - "compile GekkoFWD" script: -- GitLab From 1e7667e3efd6a69ec93ca8b970de7f574b011c6f Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 15:06:06 -0300 Subject: [PATCH 06/71] Handle CI cache between branches --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c80560ecc..64b1f510a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,7 +43,7 @@ image: jeanbez/gekkofs-forwarding:latest compile dependencies: stage: build deps cache: - key: deps-cache + key: ${CI_COMMIT_REF_SLUG} paths: - ${DEPS_INSTALL_PATH}/ script: -- GitLab From 907eadc283aead3b4b56627faf7b3714944118ab Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 15:23:34 -0300 Subject: [PATCH 07/71] Try to fix missing library in CI --- .gitlab-ci.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 64b1f510a..7c5b264ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,6 @@ stages: - build deps - build - test - - test forwarding variables: DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" @@ -215,7 +214,7 @@ test FWD wr: - "${LOG_PATH}" test FWD directories: - stage: test forwarding + stage: test dependencies: - "compile GekkoFWD" script: @@ -231,7 +230,7 @@ test FWD directories: - "${LOG_PATH}" test FWD truncate: - stage: test forwarding + stage: test dependencies: - "compile GekkoFWD" script: @@ -247,7 +246,7 @@ test FWD truncate: - "${LOG_PATH}" test FWD path resolution: - stage: test forwarding + stage: test dependencies: - "compile GekkoFWD" script: @@ -263,7 +262,7 @@ test FWD path resolution: - "${LOG_PATH}" test FWD lseek: - stage: test forwarding + stage: test dependencies: - "compile GekkoFWD" script: -- GitLab From 08ba93337d6d227f654f7d02537f428242caa88a Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 15:28:20 -0300 Subject: [PATCH 08/71] Try to fix missing library in CI --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7c5b264ee..2f4d1a112 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -198,7 +198,7 @@ test lseek: - "${LOG_PATH}" test FWD wr: - stage: test forwarding + stage: test dependencies: - "compile GekkoFWD" script: -- GitLab From c089954933c89fcc8fd42ac3ba3f094dfba9c913 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 16:14:36 -0300 Subject: [PATCH 09/71] Try to fix missing library in CI --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2f4d1a112..27f921d10 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,10 +19,10 @@ variables: INTEGRATION_TESTS_FWD_BIN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration" INTEGRATION_TESTS_FWD_RUN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/run" - TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build" + TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test-gkfs/build" PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" - TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test/build" + TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test-gkfwd/build" PYTEST_FWD: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" LOG_PATH: "${CI_PROJECT_DIR}/logs" -- GitLab From 512687a509cc906703d47e9ca39add050335acc4 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 17:52:03 -0300 Subject: [PATCH 10/71] Try to fix missing library in CI --- .gitlab-ci.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27f921d10..cb1a10377 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,10 +19,10 @@ variables: INTEGRATION_TESTS_FWD_BIN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration" INTEGRATION_TESTS_FWD_RUN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/run" - TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test-gkfs/build" + TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/gkfs-test/build" PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" - TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test-gkfwd/build" + TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/gkfwd-test/build" PYTEST_FWD: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" LOG_PATH: "${CI_PROJECT_DIR}/logs" @@ -42,7 +42,7 @@ image: jeanbez/gekkofs-forwarding:latest compile dependencies: stage: build deps cache: - key: ${CI_COMMIT_REF_SLUG} + key: deps-build paths: - ${DEPS_INSTALL_PATH}/ script: @@ -201,6 +201,7 @@ test FWD wr: stage: test dependencies: - "compile GekkoFWD" + - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - cp agios/* /tmp @@ -217,6 +218,7 @@ test FWD directories: stage: test dependencies: - "compile GekkoFWD" + - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - cp agios/* /tmp @@ -233,6 +235,7 @@ test FWD truncate: stage: test dependencies: - "compile GekkoFWD" + - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - cp agios/* /tmp @@ -249,6 +252,7 @@ test FWD path resolution: stage: test dependencies: - "compile GekkoFWD" + - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - cp agios/* /tmp @@ -265,6 +269,7 @@ test FWD lseek: stage: test dependencies: - "compile GekkoFWD" + - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - cp agios/* /tmp -- GitLab From 5ceca9eb3308d06cbb22959adc3d13024e422224 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 00:01:52 -0300 Subject: [PATCH 11/71] Working to fix CI issue --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cb1a10377..08b2c21eb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,10 +19,10 @@ variables: INTEGRATION_TESTS_FWD_BIN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration" INTEGRATION_TESTS_FWD_RUN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/run" - TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/gkfs-test/build" + TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build-gkfs" PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" - TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/gkfwd-test/build" + TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test/build-gkfwd" PYTEST_FWD: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" LOG_PATH: "${CI_PROJECT_DIR}/logs" -- GitLab From fabb781e6b7da7da298c492eb49dbe751a866f42 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 00:17:06 -0300 Subject: [PATCH 12/71] Working to fix CI issue --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 08b2c21eb..7bba1c91d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -200,6 +200,7 @@ test lseek: test FWD wr: stage: test dependencies: + - "compile dependencies" - "compile GekkoFWD" - "tests GekkoFWD" script: -- GitLab From 3a24cfff9bf11e13feb095870ffb16507d8622e4 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 00:46:43 -0300 Subject: [PATCH 13/71] Include dependency on CI tests --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7bba1c91d..ad5de816f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -218,6 +218,7 @@ test FWD wr: test FWD directories: stage: test dependencies: + - "compile dependencies" - "compile GekkoFWD" - "tests GekkoFWD" script: @@ -235,6 +236,7 @@ test FWD directories: test FWD truncate: stage: test dependencies: + - "compile dependencies" - "compile GekkoFWD" - "tests GekkoFWD" script: @@ -252,6 +254,7 @@ test FWD truncate: test FWD path resolution: stage: test dependencies: + - "compile dependencies" - "compile GekkoFWD" - "tests GekkoFWD" script: @@ -269,6 +272,7 @@ test FWD path resolution: test FWD lseek: stage: test dependencies: + - "compile dependencies" - "compile GekkoFWD" - "tests GekkoFWD" script: -- GitLab From abb050c462a097c8080ac194d72f06f7cd633bea Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 01:30:52 -0300 Subject: [PATCH 14/71] Temporarily remove AGIOS from tests --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad5de816f..d2d998a09 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,7 +93,6 @@ compile GekkoFWD: -DGKFS_INSTALL_TESTS:BOOL=ON -DRPC_PROTOCOL="ofi+sockets" -DENABLE_FORWARDING:BOOL=ON - -DENABLE_AGIOS:BOOL=ON -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} -DCMAKE_INSTALL_PREFIX=${INSTALL_FWD_PATH} ${CI_PROJECT_DIR} -- GitLab From 00f2275b58ae740038f9c5173f29444f9d214817 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 02:06:34 -0300 Subject: [PATCH 15/71] Enable AGIOS integration for GekkoFWD --- .gitlab-ci.yml | 1 + src/daemon/daemon.cpp | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d2d998a09..ad5de816f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,6 +93,7 @@ compile GekkoFWD: -DGKFS_INSTALL_TESTS:BOOL=ON -DRPC_PROTOCOL="ofi+sockets" -DENABLE_FORWARDING:BOOL=ON + -DENABLE_AGIOS:BOOL=ON -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} -DCMAKE_INSTALL_PREFIX=${INSTALL_FWD_PATH} ${CI_PROJECT_DIR} diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 187aa2926..59e9df6ef 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -162,6 +162,20 @@ void init_environment() { throw; } + #ifdef GKFS_ENABLE_FORWARDING + GKFS_DATA->spdlogger()->debug("{}() Enable I/O forwarding mode", __func__); + #endif + + #ifdef GKFS_ENABLE_AGIOS + // Initialize AGIOS scheduler + GKFS_DATA->spdlogger()->debug("{}() Initializing AGIOS scheduler: '{}'", __func__, "/tmp/agios.conf"); + try { + agios_initialize(); + } catch (const std::exception & e) { + GKFS_DATA->spdlogger()->error("{}() Failed to initialize AGIOS scheduler: {}", __func__, e.what()); + throw; + } + #endif // Initialize data backend std::string chunk_storage_path = GKFS_DATA->rootdir() + "/data/chunks"s; GKFS_DATA->spdlogger()->debug("{}() Initializing storage backend: '{}'", __func__, chunk_storage_path); -- GitLab From bde4b1fd39e02a7d9a058e66882db39f408e6f58 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 14:21:56 -0300 Subject: [PATCH 16/71] Include forwarding test support in the new test harness --- CMakeLists.txt | 9 + src/client/preload.cpp | 6 +- tests/integration/conftest.py | 22 +- tests/integration/harness/cli.py | 8 + tests/integration/harness/gkfs.py | 435 ++++++++++++++++++++++++ tests/integration/pytest.ini.in | 2 +- tests/integration/pytest.install.ini.in | 2 +- 7 files changed, 477 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c515b50bd..8293bb122 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -232,6 +232,15 @@ if(GKFS_BUILD_TESTS) message(STATUS "[gekkofs] Preparing tests...") set(GKFS_TESTS_INTERFACE "lo" CACHE STRING "Network interface to use when running tests (default: lo)") message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}") + + message(STATUS "[gekkofs] Check for forwarding tests...") + if(ENABLE_FORWARDING) + set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)") + else() + set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)") + endif() + message(STATUS "[gekkofs] Forwarding tests: ${GKFS_TESTS_FORWARDING}") + add_subdirectory(tests) else() unset(GKFS_TESTS_INTERFACE CACHE) diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 14ec65f5c..4f878c1ca 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -135,13 +135,17 @@ void *forwarding_mapper(void *p) { gkfs::util::load_forwarding_map(); LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); - } catch (std::exception& e){ + } catch (std::exception& e) { exit_error_msg(EXIT_FAILURE, fmt::format("Unable set the forwarding host '{}'", e.what())); + + return; } // Sleeps for 10 seconds sleep(10); } + + return; } #endif diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 91d1ffe2a..6039ef0f7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -19,7 +19,7 @@ from pathlib import Path from harness.logger import logger, initialize_logging, finalize_logging from harness.cli import add_cli_options, set_default_log_formatter from harness.workspace import Workspace, FileCreator -from harness.gkfs import Daemon, Client, ShellClient +from harness.gkfs import Daemon, Client, ShellClient, FwdDaemon, FwdClient, ShellFwdClient from harness.reporter import report_test_status, report_test_headline, report_assertion_pass def pytest_configure(config): @@ -87,27 +87,41 @@ def gkfs_daemon(test_workspace, request): interface = request.config.getoption('--interface') - daemon = Daemon(interface, test_workspace) + if request.config.getoption('--forwarding') == 'ON': + daemon = FwdDaemon(interface, test_workspace) + else: + daemon = Daemon(interface, test_workspace) + yield daemon.run() daemon.shutdown() @pytest.fixture -def gkfs_client(test_workspace): +def gkfs_client(test_workspace, request): """ Sets up a gekkofs client environment so that operations (system calls, library calls, ...) can be requested from a co-running daemon. """ + interface = request.config.getoption('--interface') + + if request.config.getoption('--forwarding') == 'ON': + return FwdClient(test_workspace) + return Client(test_workspace) @pytest.fixture -def gkfs_shell(test_workspace): +def gkfs_shell(test_workspace, request): """ Sets up a gekkofs environment so that shell commands (stat, ls, mkdir, etc.) can be issued to a co-running daemon. """ + interface = request.config.getoption('--interface') + + if request.config.getoption('--forwarding') == 'ON': + return ShellFwdClient(test_workspace) + return ShellClient(test_workspace) @pytest.fixture diff --git a/tests/integration/harness/cli.py b/tests/integration/harness/cli.py index 3144c9b8c..bd2b10966 100644 --- a/tests/integration/harness/cli.py +++ b/tests/integration/harness/cli.py @@ -33,6 +33,14 @@ def add_cli_options(parser): help="network interface used for communications (default: 'lo')." ) + parser.addoption( + '--forwarding', + action='store', + type=str, + default='ON', + help="enable the forwarding mode (default: 'OFF')." + ) + parser.addoption( "--bin-dir", action='append', diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index 36abe40ec..738b51b6a 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -26,6 +26,7 @@ gkfs_daemon_cmd = 'gkfs_daemon' gkfs_client_cmd = 'gkfs.io' gkfs_client_lib_file = 'libgkfs_intercept.so' gkfs_hosts_file = 'gkfs_hosts.txt' +gkfs_forwarding_map_file = 'gkfs_forwarding.map' gkfs_daemon_log_file = 'gkfs_daemon.log' gkfs_daemon_log_level = '100' gkfs_client_log_file = 'gkfs_client.log' @@ -582,3 +583,437 @@ class ShellClient: @property def cwd(self): return self._workspace.twd + +class FwdDaemon: + def __init__(self, interface, workspace): + + self._address = get_ephemeral_address(interface) + self._workspace = workspace + + self._cmd = sh.Command(gkfs_daemon_cmd, self._workspace.bindirs) + self._env = os.environ.copy() + + libdirs = ':'.join( + filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + + [str(p) for p in self._workspace.libdirs])) + + self._patched_env = { + 'LD_LIBRARY_PATH' : libdirs, + 'GKFS_HOSTS_FILE' : self.cwd / gkfs_hosts_file, + 'GKFS_DAEMON_LOG_PATH' : self.logdir / gkfs_daemon_log_file, + 'GKFS_LOG_LEVEL' : gkfs_daemon_log_level + } + self._env.update(self._patched_env) + + def run(self): + + args = [ '--mountdir', self.mountdir, + '--rootdir', self.rootdir, + '-l', self._address ] + + logger.debug(f"spawning daemon") + logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, args))) + logger.debug(f"patched env:\n{pformat(self._patched_env)}") + + self._proc = self._cmd( + args, + _env=self._env, +# _out=sys.stdout, +# _err=sys.stderr, + _bg=True, + ) + + logger.debug(f"daemon process spawned (PID={self._proc.pid})") + logger.debug("waiting for daemon to be ready") + + try: + self.wait_until_active(self._proc.pid, 10.0) + except Exception as ex: + logger.error(f"daemon initialization failed: {ex}") + + # if the daemon initialized correctly but took longer than + # `timeout`, we may be leaving running processes behind + if _process_exists(self._proc.pid): + self.shutdown() + + logger.critical(f"daemon was shut down, what is ex? {ex.__repr__}?") + raise ex + + logger.debug("daemon is ready") + + return self + + def wait_until_active(self, pid, timeout, max_lines=50): + """ + Waits until a GKFS daemon is active or until a certain timeout + has expired. Checks if the daemon is running by searching its + log for a pre-defined readiness message. + + Parameters + ---------- + pid: `int` + The PID of the daemon process to wait for. + + timeout: `number` + The number of seconds to wait for + + max_lines: `int` + The maximum number of log lines to check for a match. + """ + + gkfs_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' + + init_time = perf_counter() + + while perf_counter() - init_time < timeout: + try: + logger.debug(f"checking log file") + with open(self.logdir / gkfs_daemon_log_file) as log: + for line in islice(log, max_lines): + if re.search(gkfs_daemon_active_log_pattern, line) is not None: + return + except FileNotFoundError: + # Log is missing, the daemon might have crashed... + logger.debug(f"daemon log file missing, checking if daemon is alive...") + + pid=self._proc.pid + + if not _process_exists(pid): + raise RuntimeError(f"process {pid} is not running") + + # ... or it might just be lazy. let's give it some more time + logger.debug(f"daemon {pid} found, retrying...") + + raise RuntimeError("initialization timeout exceeded") + + def shutdown(self): + logger.debug(f"terminating daemon") + + try: + self._proc.terminate() + err = self._proc.wait() + except sh.SignalException_SIGTERM: + pass + except Exception: + raise + + + @property + def cwd(self): + return self._workspace.twd + + @property + def rootdir(self): + return self._workspace.rootdir + + @property + def mountdir(self): + return self._workspace.mountdir + + @property + def logdir(self): + return self._workspace.logdir + + @property + def interface(self): + return self._interface + +class FwdClient: + """ + A class to represent a GekkoFS client process with a patched LD_PRELOAD. + This class allows tests to interact with the file system using I/O-related + function calls, be them system calls (e.g. read()) or glibc I/O functions + (e.g. opendir()). + """ + def __init__(self, workspace): + self._parser = IOParser() + self._workspace = workspace + self._cmd = sh.Command(gkfs_client_cmd, self._workspace.bindirs) + self._env = os.environ.copy() + + # create the forwarding map file + fwd_map_file = open(self.cwd / gkfs_forwarding_map_file, 'w') + fwd_map_file.write('{} {}\n'.format(socket.gethostname(), 0)) + fwd_map_file.close() + + libdirs = ':'.join( + filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + + [str(p) for p in self._workspace.libdirs])) + + # ensure the client interception library is available: + # to avoid running code with potentially installed libraries, + # it must be found in one (and only one) of the workspace's bindirs + preloads = [] + for d in self._workspace.bindirs: + search_path = Path(d) / gkfs_client_lib_file + if search_path.exists(): + preloads.append(search_path) + + if len(preloads) == 0: + logger.error(f'No client libraries found in the test\'s binary directories:') + pytest.exit("Aborted due to initialization error. Check test logs.") + + if len(preloads) != 1: + logger.error(f'Multiple client libraries found in the test\'s binary directories:') + for p in preloads: + logger.error(f' {p}') + logger.error(f'Make sure that only one copy of the client library is available.') + pytest.exit("Aborted due to initialization error. Check test logs.") + + self._preload_library = preloads[0] + + self._patched_env = { + 'LD_LIBRARY_PATH' : libdirs, + 'LD_PRELOAD' : self._preload_library, + 'LIBGKFS_HOSTS_FILE' : self.cwd / gkfs_hosts_file, + 'LIBGKFS_FORWARDING_MAP_FILE' : self.cwd / gkfs_forwarding_map_file, + 'LIBGKFS_LOG' : gkfs_client_log_level, + 'LIBGKFS_LOG_OUTPUT' : self._workspace.logdir / gkfs_client_log_file + } + + self._env.update(self._patched_env) + + @property + def preload_library(self): + """ + Return the preload library detected for this client + """ + + return self._preload_library + + def run(self, cmd, *args): + + logger.debug(f"running client") + logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, list(args)))) + logger.debug(f"patched env: {pformat(self._patched_env)}") + + out = self._cmd( + [ cmd ] + list(args), + _env = self._env, + # _out=sys.stdout, + # _err=sys.stderr, + ) + + logger.debug(f"command output: {out.stdout}") + return self._parser.parse(cmd, out.stdout) + + def __getattr__(self, name): + return _proxy_exec(self, name) + + @property + def cwd(self): + return self._workspace.twd + +class ShellFwdClient: + """ + A class to represent a GekkoFS shell client process. + This class allows tests to execute shell commands or scripts via bash -c + on a GekkoFS instance. + """ + + def __init__(self, workspace): + self._workspace = workspace + self._cmd = sh.Command("bash") + self._env = os.environ.copy() + + # create the forwarding map file + fwd_map_file = open(self.cwd / gkfs_forwarding_map_file, 'w') + fwd_map_file.write('{} {}\n'.format(socket.gethostname(), 0)) + fwd_map_file.close() + + libdirs = ':'.join( + filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + + [str(p) for p in self._workspace.libdirs])) + + # ensure the client interception library is available: + # to avoid running code with potentially installed libraries, + # it must be found in one (and only one) of the workspace's bindirs + preloads = [] + for d in self._workspace.bindirs: + search_path = Path(d) / gkfs_client_lib_file + if search_path.exists(): + preloads.append(search_path) + + if len(preloads) != 1: + logger.error(f'Multiple client libraries found in the test\'s binary directories:') + for p in preloads: + logger.error(f' {p}') + logger.error(f'Make sure that only one copy of the client library is available.') + pytest.exit("Aborted due to initialization error") + + self._preload_library = preloads[0] + + self._patched_env = { + 'LD_LIBRARY_PATH' : libdirs, + 'LD_PRELOAD' : self._preload_library, + 'LIBGKFS_HOSTS_FILE' : self.cwd / gkfs_hosts_file, + 'LIBGKFS_FORWARDING_MAP_FILE' : self.cwd / gkfs_forwarding_map_file, + 'LIBGKFS_LOG' : gkfs_client_log_level, + 'LIBGKFS_LOG_OUTPUT' : self._workspace.logdir / gkfs_client_log_file + } + + self._env.update(self._patched_env) + + @property + def patched_environ(self): + """ + Return the patched environment required to run a test as a string that + can be prepended to a shell command. + """ + + return ' '.join(f'{k}="{v}"' for k,v in self._patched_env.items()) + + def script(self, code, intercept_shell=True, timeout=60, timeout_signal=signal.SIGKILL): + """ + Execute a shell script passed as an argument in bash. + + For instance, the following snippet: + + mountdir = pathlib.Path('/tmp') + file01 = 'file01' + + ShellClient().script( + f''' + expected_pathname={mountdir / file01} + if [[ -e ${{expected_pathname}} ]]; + then + exit 0 + fi + exit 1 + ''') + + transforms into: + + bash -c ' + expected_pathname=/tmp/file01 + if [[ -e ${expected_pathname} ]]; + then + exit 0 + fi + exit 1 + ' + + Note that since we are using Python's f-strings, for variable + expansions to work correctly, they need to be defined with double + braces, e.g. ${{expected_pathname}}. + + Parameters + ---------- + code: `str` + The script code to be passed to 'bash -c'. + + intercept_shell: `bool` + Controls whether the shell executing the script should be + executed with LD_PRELOAD=libgkfs_intercept.so (default: True). + + timeout: `int` + How much time, in seconds, we should give the process to complete. + If the process does not finish within the timeout, it will be sent + the signal defined by `timeout_signal`. + + Default value: 60 + + timeout_signal: `int` + The signal to be sent to the process if `timeout` is not None. + + Default value: signal.SIGKILL + + Returns + ------- + A sh.RunningCommand instance that allows interacting with + the finished process. + """ + + logger.debug(f"running bash") + logger.debug(f"cmd: bash -c '{code}'") + logger.debug(f"timeout: {timeout} seconds") + logger.debug(f"timeout_signal: {signal.Signals(timeout_signal).name}") + + if intercept_shell: + logger.debug(f"patched env: {self._patched_env}") + + # 'sh' raises an exception if the return code is not zero; + # since we'd rather check for return codes explictly, we + # whitelist all exit codes from 1 to 255 as 'ok' using the + # _ok_code argument + return self._cmd('-c', + code, + _env = (self._env if intercept_shell else os.environ), + # _out=sys.stdout, + # _err=sys.stderr, + _timeout=timeout, + _timeout_signal=timeout_signal, + # _ok_code=list(range(0, 256)) + ) + + def run(self, cmd, *args, timeout=60, timeout_signal=signal.SIGKILL): + """ + Execute a shell command with arguments. + + For example, the following snippet: + + mountdir = pathlib.Path('/tmp') + file01 = 'file01' + + ShellClient().stat('--terse', mountdir / file01) + + transforms into: + + bash -c 'stat --terse /tmp/file01' + + Parameters: + ----------- + cmd: `str` + The command to execute. + + args: `list` + The list of arguments for the command. + + timeout: `number` + How much time, in seconds, we should give the process to complete. + If the process does not finish within the timeout, it will be sent + the signal defined by `timeout_signal`. + + Default value: 60 + + timeout_signal: `int` + The signal to be sent to the process if `timeout` is not None. + + Default value: signal.SIGKILL + + Returns + ------- + A ShellCommand instance that allows interacting with the finished + process. Note that ShellCommand wraps sh.RunningCommand and adds s + extra properties to it. + """ + + bash_c_args = f"{cmd} {' '.join(str(a) for a in args)}" + logger.debug(f"running bash") + logger.debug(f"cmd: bash -c '{bash_c_args}'") + logger.debug(f"timeout: {timeout} seconds") + logger.debug(f"timeout_signal: {signal.Signals(timeout_signal).name}") + logger.debug(f"patched env:\n{pformat(self._patched_env)}") + + # 'sh' raises an exception if the return code is not zero; + # since we'd rather check for return codes explictly, we + # whitelist all exit codes from 1 to 255 as 'ok' using the + # _ok_code argument + proc = self._cmd('-c', + bash_c_args, + _env = self._env, + # _out=sys.stdout, + # _err=sys.stderr, + _timeout=timeout, + _timeout_signal=timeout_signal, + # _ok_code=list(range(0, 256)) + ) + + return ShellCommand(cmd, proc) + + def __getattr__(self, name): + return _proxy_exec(self, name) + + @property + def cwd(self): + return self._workspace.twd diff --git a/tests/integration/pytest.ini.in b/tests/integration/pytest.ini.in index d7901fa8f..3edd1784c 100644 --- a/tests/integration/pytest.ini.in +++ b/tests/integration/pytest.ini.in @@ -13,5 +13,5 @@ [pytest] testpaths = @CMAKE_CURRENT_SOURCE_DIR@ -addopts = @PYTEST_BINDIR_ARGS@ @PYTEST_LIBDIR_ARGS@ --interface=@GKFS_TESTS_INTERFACE@ +addopts = @PYTEST_BINDIR_ARGS@ @PYTEST_LIBDIR_ARGS@ --interface=@GKFS_TESTS_INTERFACE@ --forwarding=@GKFS_TESTS_FORWARDING@ enable_assertion_pass_hook = true diff --git a/tests/integration/pytest.install.ini.in b/tests/integration/pytest.install.ini.in index 23e87ec25..0d6889ef7 100644 --- a/tests/integration/pytest.install.ini.in +++ b/tests/integration/pytest.install.ini.in @@ -13,5 +13,5 @@ [pytest] testpaths = @CMAKE_INSTALL_FULL_DATAROOTDIR@/gkfs/tests -addopts = --bin-dir=@CMAKE_INSTALL_FULL_BINDIR@ --bin-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --lib-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --interface=@GKFS_TESTS_INTERFACE@ +addopts = --bin-dir=@CMAKE_INSTALL_FULL_BINDIR@ --bin-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --lib-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --interface=@GKFS_TESTS_FORWARDING@ --forwarding=@ENABLE_FORWARDING@ enable_assertion_pass_hook = true -- GitLab From 179fdc33acd267b8bdf8afe7160a455c3bd15b49 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 14:34:05 -0300 Subject: [PATCH 17/71] Organize the CI and fix small bug --- .gitlab-ci.yml | 22 +++++++++++++++++----- src/client/preload.cpp | 4 ---- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad5de816f..97d22d45c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ stages: - build deps - build - test + - test forwarding variables: DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" @@ -137,6 +138,17 @@ integration tests: paths: - "${INTEGRATION_TESTS_RUN_PATH}" +integration GekkoFWD tests: + stage: test GekkoFWD + script: + - mkdir -p ${INTEGRATION_TESTS_FWD_RUN_PATH} + - cd ${INTEGRATION_TESTS_FWD_BIN_PATH} + - TMPDIR=${INTEGRATION_TESTS_FWD_RUN_PATH} unbuffer ${PYTEST} -v | tee ${INTEGRATION_TESTS_FWD_RUN_PATH}/session.log + artifacts: + when: on_failure + paths: + - "${INTEGRATION_TESTS_FWD_RUN_PATH}" + test wr: stage: test script: @@ -198,7 +210,7 @@ test lseek: - "${LOG_PATH}" test FWD wr: - stage: test + stage: test forwarding dependencies: - "compile dependencies" - "compile GekkoFWD" @@ -216,7 +228,7 @@ test FWD wr: - "${LOG_PATH}" test FWD directories: - stage: test + stage: test forwarding dependencies: - "compile dependencies" - "compile GekkoFWD" @@ -234,7 +246,7 @@ test FWD directories: - "${LOG_PATH}" test FWD truncate: - stage: test + stage: test forwarding dependencies: - "compile dependencies" - "compile GekkoFWD" @@ -252,7 +264,7 @@ test FWD truncate: - "${LOG_PATH}" test FWD path resolution: - stage: test + stage: test forwarding dependencies: - "compile dependencies" - "compile GekkoFWD" @@ -270,7 +282,7 @@ test FWD path resolution: - "${LOG_PATH}" test FWD lseek: - stage: test + stage: test forwarding dependencies: - "compile dependencies" - "compile GekkoFWD" diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 4f878c1ca..7d1cfd3bc 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -137,15 +137,11 @@ void *forwarding_mapper(void *p) { LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); } catch (std::exception& e) { exit_error_msg(EXIT_FAILURE, fmt::format("Unable set the forwarding host '{}'", e.what())); - - return; } // Sleeps for 10 seconds sleep(10); } - - return; } #endif -- GitLab From 1f7626781bdfa885ad321fff518f4083ead660ef Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 14:36:01 -0300 Subject: [PATCH 18/71] Fix typo in CI .yaml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 97d22d45c..bf3594a52 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -139,7 +139,7 @@ integration tests: - "${INTEGRATION_TESTS_RUN_PATH}" integration GekkoFWD tests: - stage: test GekkoFWD + stage: test forwarding script: - mkdir -p ${INTEGRATION_TESTS_FWD_RUN_PATH} - cd ${INTEGRATION_TESTS_FWD_BIN_PATH} -- GitLab From 5527605db586219d0fa083baab8691b8923bb192 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 18:31:10 -0300 Subject: [PATCH 19/71] Fix typo in variable name for the interface --- .gitlab-ci.yml | 2 +- tests/integration/pytest.install.ini.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf3594a52..6b52e0245 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -138,7 +138,7 @@ integration tests: paths: - "${INTEGRATION_TESTS_RUN_PATH}" -integration GekkoFWD tests: +integration FWD tests: stage: test forwarding script: - mkdir -p ${INTEGRATION_TESTS_FWD_RUN_PATH} diff --git a/tests/integration/pytest.install.ini.in b/tests/integration/pytest.install.ini.in index 0d6889ef7..a8668942b 100644 --- a/tests/integration/pytest.install.ini.in +++ b/tests/integration/pytest.install.ini.in @@ -13,5 +13,5 @@ [pytest] testpaths = @CMAKE_INSTALL_FULL_DATAROOTDIR@/gkfs/tests -addopts = --bin-dir=@CMAKE_INSTALL_FULL_BINDIR@ --bin-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --lib-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --interface=@GKFS_TESTS_FORWARDING@ --forwarding=@ENABLE_FORWARDING@ +addopts = --bin-dir=@CMAKE_INSTALL_FULL_BINDIR@ --bin-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --lib-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --interface=@GKFS_TESTS_INTERFACE@ --forwarding=@ENABLE_FORWARDING@ enable_assertion_pass_hook = true -- GitLab From 4ddc00c39bbac2ae4a67febc4eb640eaaadfeaca Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 19 May 2020 12:41:41 -0300 Subject: [PATCH 20/71] Remove sudo from Dockerfile --- docker/build_env.docker | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/build_env.docker b/docker/build_env.docker index ef83cbaa9..346807b86 100644 --- a/docker/build_env.docker +++ b/docker/build_env.docker @@ -80,7 +80,6 @@ RUN yum -y -q update && yum -y -q install \ automake \ cmake \ cmake3 \ - sudo \ gcc \ gcc-c++ \ openssl-devel \ -- GitLab From 78083b9085246aa1c880e13d4a88d70f422acee7 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 19 May 2020 12:50:35 -0300 Subject: [PATCH 21/71] Fix typo --- src/client/gkfs_functions.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 0e1037a2b..a2fd4a126 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -414,7 +414,7 @@ ssize_t gkfs_write(int fd, const void* buf, size_t count) { ssize_t gkfs_pwritev(int fd, const struct iovec* iov, int iovcnt, off_t offset) { auto file = CTX->file_map()->get(fd); - auto pos = offset; // keep truck of current position + auto pos = offset; // keep track of current position ssize_t written = 0; ssize_t ret; for (int i = 0; i < iovcnt; ++i) { @@ -488,7 +488,7 @@ ssize_t gkfs_read(int fd, void* buf, size_t count) { ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) { auto file = CTX->file_map()->get(fd); - auto pos = offset; // keep truck of current position + auto pos = offset; // keep track of current position ssize_t read = 0; ssize_t ret; for (int i = 0; i < iovcnt; ++i) { -- GitLab From c8e6c529dfbaf78732fa4a4996870a28f2f3113c Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 19 May 2020 12:53:22 -0300 Subject: [PATCH 22/71] Rename ENABLE_FORWARDING to GKFS_ENABLE_FORWARDING --- .gitlab-ci.yml | 4 ++-- CMakeLists.txt | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6b52e0245..4f60e8730 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,8 +93,8 @@ compile GekkoFWD: -DGKFS_BUILD_TESTS:BOOL=ON -DGKFS_INSTALL_TESTS:BOOL=ON -DRPC_PROTOCOL="ofi+sockets" - -DENABLE_FORWARDING:BOOL=ON - -DENABLE_AGIOS:BOOL=ON + -DGFKS_ENABLE_FORWARDING:BOOL=ON + -DGFKS_ENABLE_AGIOS:BOOL=ON -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} -DCMAKE_INSTALL_PREFIX=${INSTALL_FWD_PATH} ${CI_PROJECT_DIR} diff --git a/CMakeLists.txt b/CMakeLists.txt index 8293bb122..fcc8288ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,17 +142,17 @@ if(ENABLE_CLIENT_LOG) add_definitions(-DGKFS_ENABLE_LOGGING) endif() message(STATUS "[gekkofs] Client logging output: ${ENABLE_CLIENT_LOG}") -option(ENABLE_FORWARDING "Enable forwarding mode" OFF) -if(ENABLE_FORWARDING) +option(GKFS_ENABLE_FORWARDING "Enable forwarding mode" OFF) +if(GKFS_ENABLE_FORWARDING) add_definitions(-DGKFS_ENABLE_FORWARDING) endif() -message(STATUS "[gekkofs] Forwarding mode: ${ENABLE_FORWARDING}") +message(STATUS "[gekkofs] Forwarding mode: ${GKFS_ENABLE_FORWARDING}") -option(ENABLE_AGIOS "Enable AGIOS scheduling library" OFF) -if(ENABLE_AGIOS) +option(GKFS_ENABLE_AGIOS "Enable AGIOS scheduling library" OFF) +if(GKFS_ENABLE_AGIOS) add_definitions(-DGKFS_ENABLE_AGIOS) endif() -message(STATUS "[gekkofs] AGIOS scheduling: ${ENABLE_AGIOS}") +message(STATUS "[gekkofs] AGIOS scheduling: ${GKFS_ENABLE_AGIOS}") set(CLIENT_LOG_MESSAGE_SIZE 1024 CACHE STRING "Maximum size of a log message in the client library") add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${CLIENT_LOG_MESSAGE_SIZE}) -- GitLab From cca20c27166e1d2bf9c2bbce875053f404a8a8f3 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 19 May 2020 12:53:42 -0300 Subject: [PATCH 23/71] Follow Gekko coding conventions --- src/client/preload.cpp | 2 +- src/daemon/handler/srv_data.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 7d1cfd3bc..9959254bc 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -129,7 +129,7 @@ void init_ld_environment_() { } #ifdef GKFS_ENABLE_FORWARDING -void *forwarding_mapper(void *p) { +void *forwarding_mapper(void* p) { while (forwarding_running) { try { gkfs::util::load_forwarding_map(); diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index 2c3226abb..4eae87c9b 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -644,11 +644,11 @@ static hg_return_t rpc_srv_get_chunk_stat(hg_handle_t handle) { DEFINE_MARGO_RPC_HANDLER(rpc_srv_get_chunk_stat) #ifdef GKFS_ENABLE_AGIOS -void *agios_eventual_callback(int64_t request_id, void *info) { +void *agios_eventual_callback(int64_t request_id, void* info) { GKFS_DATA->spdlogger()->debug("{}() custom callback request {} is ready", __func__, request_id); ABT_eventual_set((ABT_eventual) info, &request_id, sizeof(int64_t)); - + return 0; } #endif -- GitLab From 44f20ee884652fe9d721607db4398f932fcc1487 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 11:30:48 -0300 Subject: [PATCH 24/71] Create a forwarding mode in GekkoFS --- .gitlab-ci.yml | 155 +++++++++++++++++++++++++---- CMake/FindAGIOS.cmake | 18 ++++ CMakeLists.txt | 12 +++ agios/access_times.func | 11 ++ agios/agios.conf | 92 +++++++++++++++++ docker/build_env.docker | 5 + docker/debian_build_env.docker | 2 + include/client/env.hpp | 3 + include/client/preload_context.hpp | 5 + include/client/preload_util.hpp | 1 + include/config.hpp | 1 + include/daemon/scheduler/agios.hpp | 15 +++ include/global/rpc/distributor.hpp | 17 ++++ scripts/compile_dep.sh | 20 +++- scripts/dl_dep.sh | 13 ++- src/client/preload.cpp | 47 +++++++++ src/client/preload_context.cpp | 9 +- src/client/preload_util.cpp | 68 ++++++++++++- src/daemon/CMakeLists.txt | 4 + src/daemon/daemon.cpp | 19 ++++ src/daemon/handler/srv_data.cpp | 82 +++++++++++++++ src/daemon/scheduler/agios.cpp | 10 ++ src/global/rpc/distributor.cpp | 27 ++++- 23 files changed, 607 insertions(+), 29 deletions(-) create mode 100644 CMake/FindAGIOS.cmake create mode 100644 agios/access_times.func create mode 100644 agios/agios.conf create mode 100644 include/daemon/scheduler/agios.hpp create mode 100644 src/daemon/scheduler/agios.cpp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1a2457ca3..9b1ac1266 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,28 +2,44 @@ stages: - download deps - build deps - build + - build forwarding - test + - test forwarding variables: - DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" - DEPS_INSTALL_PATH: "${CI_PROJECT_DIR}/deps/install" - DEPS_COMMIT: "${CI_PROJECT_DIR}/deps/install/gkfs_deps_commit" - BUILD_PATH: "${CI_PROJECT_DIR}/gkfs/build" - INSTALL_PATH: "${CI_PROJECT_DIR}/gkfs/install" - INTEGRATION_TESTS_BIN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration" - INTEGRATION_TESTS_RUN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/run" - TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build" - PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" - LOG_PATH: "${CI_PROJECT_DIR}/logs" - LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/deps/install/lib;${CI_PROJECT_DIR}/deps/install/lib64" + DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" + DEPS_INSTALL_PATH: "${CI_PROJECT_DIR}/deps/install" + DEPS_COMMIT: "${CI_PROJECT_DIR}/deps/install/gkfs_deps_commit" + + BUILD_PATH: "${CI_PROJECT_DIR}/gkfs/build" + INSTALL_PATH: "${CI_PROJECT_DIR}/gkfs/install" + INTEGRATION_TESTS_BIN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration" + INTEGRATION_TESTS_RUN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/run" + + BUILD_FWD_PATH: "${CI_PROJECT_DIR}/gkfwd/build" + INSTALL_FWD_PATH: "${CI_PROJECT_DIR}/gkfwd/install" + INTEGRATION_TESTS_FWD_BIN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration" + INTEGRATION_TESTS_FWD_RUN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/run" + + TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/gkfs/build" + PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" + + TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test/gkfwd/build" + PYTEST_FWD: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" + + LOG_PATH: "${CI_PROJECT_DIR}/logs" + + LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/deps/install/lib;${CI_PROJECT_DIR}/deps/install/lib64" # Configuration variables - GKFS_LOG_LEVEL: "100" - GKFS_DAEMON_LOG_PATH: "${CI_PROJECT_DIR}/logs/daemon.log" - LIBGKFS_LOG: "all" - LIBGKFS_LOG_OUTPUT: "${CI_PROJECT_DIR}/logs/gkfs_client.log" - GIT_SUBMODULE_STRATEGY: recursive + GKFS_LOG_LEVEL: "100" + GKFS_DAEMON_LOG_PATH: "${CI_PROJECT_DIR}/logs/daemon.log" + LIBGKFS_LOG: "all" + LIBGKFS_LOG_OUTPUT: "${CI_PROJECT_DIR}/logs/gkfs_client.log" + GIT_SUBMODULE_STRATEGY: recursive -image: gekkofs/gekkofs:build_env-0.8.0 +# Temporary new image file +# image: gekkofs/gekkofs:build_env-0.8.0 +image: jeanbez/gekkofs-forwarding:latest compile dependencies: stage: build deps @@ -65,6 +81,29 @@ compile GekkoFS: paths: - ${INSTALL_PATH} +compile GekkoFWD: + stage: build + dependencies: + - "compile dependencies" + script: + - mkdir -p ${BUILD_FWD_PATH} && cd ${BUILD_FWD_PATH} + - cmake + -Wdev + -Wdeprecate + -DCMAKE_BUILD_TYPE=Debug + -DGKFS_BUILD_TESTS:BOOL=ON + -DGKFS_INSTALL_TESTS:BOOL=ON + -DRPC_PROTOCOL="ofi+sockets" + -DENABLE_FORWARDING:BOOL=ON \ + -DENABLE_AGIOS:BOOL=ON \ + -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} + -DCMAKE_INSTALL_PREFIX=${INSTALL_FWD_PATH} + ${CI_PROJECT_DIR} + - make -j$(nproc) install + artifacts: + paths: + - ${INSTALL_FWD_PATH} + compile tests: stage: build dependencies: @@ -147,3 +186,85 @@ test lseek: when: on_failure paths: - "${LOG_PATH}" + +test wr: + stage: test forwarding + dependencies: + - "compile GekkoFWD" + script: + - mkdir -p "${LOG_PATH}" + - cp agios/* /tmp + - echo "`hostname` 0" > gkfs_forwarding.map + - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & + - sleep 4 + - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_wr + artifacts: + when: on_failure + paths: + - "${LOG_PATH}" + +test directories: + stage: test forwarding + dependencies: + - "compile GekkoFWD" + script: + - mkdir -p "${LOG_PATH}" + - cp agios/* /tmp + - echo "`hostname` 0" > gkfs_forwarding.map + - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & + - sleep 4 + - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_dir + artifacts: + when: on_failure + paths: + - "${LOG_PATH}" + +test truncate: + stage: test forwarding + dependencies: + - "compile GekkoFWD" + script: + - mkdir -p "${LOG_PATH}" + - cp agios/* /tmp + - echo "`hostname` 0" > gkfs_forwarding.map + - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & + - sleep 4 + - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_truncate + artifacts: + when: on_failure + paths: + - "${LOG_PATH}" + +test path resolution: + stage: test forwarding + dependencies: + - "compile GekkoFWD" + script: + - mkdir -p "${LOG_PATH}" + - cp agios/* /tmp + - echo "`hostname` 0" > gkfs_forwarding.map + - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & + - sleep 4 + - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_path_resolution + artifacts: + when: on_failure + paths: + - "${LOG_PATH}" + +test lseek: + stage: test forwarding + dependencies: + - "compile GekkoFWD" + script: + - mkdir -p "${LOG_PATH}" + - cp agios/* /tmp + - echo "`hostname` 0" > gkfs_forwarding.map + - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & + - sleep 4 + - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_lseek + artifacts: + when: on_failure + paths: + - "${LOG_PATH}" + + diff --git a/CMake/FindAGIOS.cmake b/CMake/FindAGIOS.cmake new file mode 100644 index 000000000..a11910d33 --- /dev/null +++ b/CMake/FindAGIOS.cmake @@ -0,0 +1,18 @@ +find_path(AGIOS_INCLUDE_DIR + NAMES agios.h +) + +find_library(AGIOS_LIBRARY + NAMES agios +) + +set(AGIOS_INCLUDE_DIRS ${AGIOS_INCLUDE_DIR}) +set(AGIOS_LIBRARIES ${AGIOS_LIBRARY}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(AGIOS DEFAULT_MSG AGIOS_LIBRARIES AGIOS_INCLUDE_DIRS) + +mark_as_advanced( + AGIOS_LIBRARY + AGIOS_INCLUDE_DIR +) \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e6d61290..c515b50bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,7 @@ find_package(Snappy REQUIRED) find_package(ZStd REQUIRED) find_package(JeMalloc) # required if rocksdb has been build with jemalloc find_package(RocksDB REQUIRED) +find_package(AGIOS REQUIRED) # margo dependencies find_package(Mercury REQUIRED) find_package(Abt REQUIRED) @@ -141,6 +142,17 @@ if(ENABLE_CLIENT_LOG) add_definitions(-DGKFS_ENABLE_LOGGING) endif() message(STATUS "[gekkofs] Client logging output: ${ENABLE_CLIENT_LOG}") +option(ENABLE_FORWARDING "Enable forwarding mode" OFF) +if(ENABLE_FORWARDING) + add_definitions(-DGKFS_ENABLE_FORWARDING) +endif() +message(STATUS "[gekkofs] Forwarding mode: ${ENABLE_FORWARDING}") + +option(ENABLE_AGIOS "Enable AGIOS scheduling library" OFF) +if(ENABLE_AGIOS) + add_definitions(-DGKFS_ENABLE_AGIOS) +endif() +message(STATUS "[gekkofs] AGIOS scheduling: ${ENABLE_AGIOS}") set(CLIENT_LOG_MESSAGE_SIZE 1024 CACHE STRING "Maximum size of a log message in the client library") add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${CLIENT_LOG_MESSAGE_SIZE}) diff --git a/agios/access_times.func b/agios/access_times.func new file mode 100644 index 000000000..a6accf355 --- /dev/null +++ b/agios/access_times.func @@ -0,0 +1,11 @@ +2 +8 64 +566.8405 6784.471 +238.2483 1173.456 +145.6056 1018.143 +187.2437 -201.8793 +64 128 +571.2364 -740.0433 +234.6933 41.99119 +218.3765 -6295.179 +113.1743 2235.442 diff --git a/agios/agios.conf b/agios/agios.conf new file mode 100644 index 000000000..e0fc38d0a --- /dev/null +++ b/agios/agios.conf @@ -0,0 +1,92 @@ +library_options: +{ #ATTENTION: with my most recent modification on statistics, the prediction module is NOT WORKING, DO NOT USE IT. + #should we generate trace files during execution? + trace = false ; + + #should we trace predicted requests? (for debug purposes, this trace will not be useful for future executions) + trace_predict = false ; + + #should we make a complete trace, with all scheduler's operations, such as waiting times? (for debug purposes) + trace_full = false ; + + #should the prediction module read trace files? (if false, the Prediction Module is useless) + predict_read_traces = false ; + + #should the prediction module try to predict aggregations? (requires predict_read_traces = true) + predict_request_aggregation = false ; + + #should the prediction module create simplified traces with information (the metrics) it obtained from the real traces? + predict_write_simplified_traces = false; + + #the tolerance for arrival times difference when checking if two predicted requests are the same (in %) + prediction_time_error = 10 + + #this parameter gives the frequency with which the prediction module will redo its predicted aggregations (in number of requests that must be processed between refreshs). This is necessary because these predictions use a factor that represents the ability to overlap waiting times with processing of other requests. At initialization, this factor will be calculated from the provided trace files, but during execution it can be recalculated using measurements for this ability during the actual scheduling. If the parameter is set to -1, aggregations will not be recalculated during execution. (in number of requests) + prediction_recalculate_alpha_period = -1 + + #prefix and sufix for trace files (with path). Their names must be trace_file_prefix+"."+number+"."+trace_file_sufix, with ordered numbers (no holes) + trace_file_prefix = "/tmp/agios_tracefile" + trace_file_sufix = "out" + #prefix for simple trace files (with path). Their names will be prefix+"."+number+"."+trace_file_sufix + simple_trace_prefix = "/tmp/agios_simpletracefile" + + #parameters used by aIOLi and MLF + waiting_time = 900000 + aioli_quantum = 65536 + mlf_quantum = 8192 + + #parameter used by TW (ms) + time_window_size = 1000 #the paper proposing TW recommends 1000 for HDD and 250 for SSD. + + #file (with path) with access times functions (generated by SeRRa - http://serratoool.bitbucket.org/). Used by aIOLi to quantum assignment and by the mechanism that automatically selects the best scheduling algorithm to use. If you are using a static algorithm which is not aIOLi, this does not matter, but you need to provide it anyway. In this case, you can use the one provided as example with the library source code + access_times_func_file = "/tmp/access_times.func" + + #to how many scheduling algorithms the performance module keeps measurements. When we are changing scheduling algorithms, we may observe new measurements (through the agios_release_request function) to the previous algorithms, so we could update information we have for them. It makes no sense to have a big value for performance_values if we don't change algorithms too often + performance_values = 5 + + #how many scheduling algorithms changes we register in the stats file? + proc_algs = 1000 + + #default I/O scheduling algorithm to use (the one to be used if the previous value was set to false) + #existing algorithms (case sensitive): "MLF", "aIOLi", "SJF", "TO", "TO-agg", "SRTF", "TW", "NOOP", "DYN_TREE", "ARMED_BANDIT" (case sensitive) + # SRTF it's experimental and uses information from trace files (don't use it if these are not available) + # NOOP is the NO operation scheduling algorithm, you should not observe performance improvements by using this one + # TW only makes sense if the user is providing AGIOS correct application id for each request. Don't use it otherwise + # DYN_TREE is a dynamic scheduling algorithm which selects the best (among MLF, AIOLI, SFJ, TO, TO-AGG, and NOOP) according to a decision tree. Use it only when using AGIOS to schedule requests to parallel file system' servers. + # ARMED_BANDIT is another dynamic algorithm which periodically selects between MLF, aIOLi, SJF, TO, TO-agg, and NOOP. It keeps performance measurements for these algorithms and gives more probability to the better ones. + default_algorithm = "TO"; + + # TWINS window size, in us + twins_window_duration = 1000; + dynamic_twins = false; + # time between window size adaptation, in ms + dynamic_twins_period = 1000; + + dynamic_twins_clients = 32; + dynamic_twins_processes = 128; + + # Only relevant if default_algorithm is a dynamic one. this parameter gives the frequency with which the automatic scheduling algorithm selection will recalculate the scheduling algorithm. This selection will be done using the access pattern from this period. If -1 is provided, then the selection will be done at the beginning of execution only (using information from traces for DYN_TREE). The next parameter gives the minimum number of requests which need to happen in this period for the selection to be done (otherwise we will wait longer before recalculating). (in msec). Notice that it makes no sense to use ARMED_BANDIT without setting period and min_reqnumber, as it needs to be iteractive. + select_algorithm_period = 1000 + + select_algorithm_min_reqnumber=1 + + #also for dynamic algorithms. says if TW should be one of the options considered by the dynamic scheduler. TW requires identifying requests according to the applications they come from. If this is not possible, don't use TW! + enable_TW = false ; + + #if default_algorithm is a dynamic algorithm, you need to indicate which static algorithm to use at first (before automatically selecting the next one). Only relevant if you are using ARMED_BANDIT + starting_algorithm = "SJF" ; + + # ARMED_BANDIT parameters - only relevant if default_algorithm is ARMED_BANDIT, but need to be provided anyway. min_ab_probability gives the minimum probability given to scheduling algorithms which performed poorly. The algorithm needs to maintain some probability to accomodate changes in the access pattern. validity_window is the period of time (in msec) for which performance measurements are still valid, after this time period we discard them (so we adapt to new situations). performance_window determines how many performance measurements the ARMED_BANDIT algorithms keeps for each scheduling algorithm option (taking the average of them). Keeping a huge window takes more memory. Moreover, it is related with validity_window and select_algorithm_period as if we discard measurements too often, we will never fill the whole window. + min_ab_probability = 3 + validity_window = 360000 + performance_window = 10 +}; +user_info: +{ + #stripe size used by the library's users (in bytes). This is used for detecting the access pattern at a parallel file system server. Useless for other uses. + stripe_size = 32768 ; + + #maximum buffer size used for storing trace parts (in KB). Having a buffer avoids generating requests to the local file system, which interfere in performance. On the other hand, having a large buffer can affect performance and decrease available space for data buffer. + max_trace_buffer_size = 32768 ; + +}; diff --git a/docker/build_env.docker b/docker/build_env.docker index 46094c08e..ef83cbaa9 100644 --- a/docker/build_env.docker +++ b/docker/build_env.docker @@ -25,6 +25,7 @@ RUN yum -y -q update && yum -y -q install \ gcc \ gcc-c++ \ openssl-devel \ + libconfig-devel \ # Mercury dependencies libtool \ libtool-ltdl-devel \ @@ -34,6 +35,7 @@ RUN yum -y -q update && yum -y -q install \ snappy-devel \ libzstd-devel \ lz4-devel \ + bzip2 \ bzip2-devel \ # ada-fs requires C++ 14 devtoolset-8-gcc \ @@ -78,9 +80,11 @@ RUN yum -y -q update && yum -y -q install \ automake \ cmake \ cmake3 \ + sudo \ gcc \ gcc-c++ \ openssl-devel \ + libconfig-devel \ # Mercury dependencies libtool \ libtool-ltdl-devel \ @@ -90,6 +94,7 @@ RUN yum -y -q update && yum -y -q install \ snappy-devel \ libzstd-devel \ lz4-devel \ + bzip2 \ bzip2-devel \ # ada-fs requires C++ 14 devtoolset-8-gcc \ diff --git a/docker/debian_build_env.docker b/docker/debian_build_env.docker index b697b49e9..743e10d22 100644 --- a/docker/debian_build_env.docker +++ b/docker/debian_build_env.docker @@ -19,6 +19,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ gcc \ g++ \ + # AGIOS dependencies + libconfig-dev \ # Mercury dependencies libltdl-dev \ lbzip2 \ diff --git a/include/client/env.hpp b/include/client/env.hpp index e438ab7f4..0555371f7 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -33,6 +33,9 @@ static constexpr auto LOG_OUTPUT = ADD_PREFIX("LOG_OUTPUT"); static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); static constexpr auto CWD = ADD_PREFIX("CWD"); static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); +#ifdef GKFS_ENABLE_FORWARDING +static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE"); +#endif } // namespace env } // namespace gkfs diff --git a/include/client/preload_context.hpp b/include/client/preload_context.hpp index 0eb43be26..616a92966 100644 --- a/include/client/preload_context.hpp +++ b/include/client/preload_context.hpp @@ -82,6 +82,7 @@ private: std::vector hosts_; uint64_t local_host_id_; + uint64_t fwd_host_id_; bool interception_enabled_; @@ -123,6 +124,10 @@ public: void local_host_id(uint64_t id); + uint64_t fwd_host_id() const; + + void fwd_host_id(uint64_t id); + RelativizeStatus relativize_fd_path(int dirfd, const char* raw_path, std::string& relative_path, diff --git a/include/client/preload_util.hpp b/include/client/preload_util.hpp index 37cf4fc51..f2e000ddd 100644 --- a/include/client/preload_util.hpp +++ b/include/client/preload_util.hpp @@ -62,6 +62,7 @@ int metadata_to_stat(const std::string& path, const gkfs::metadata::Metadata& md std::vector> load_hostfile(const std::string& lfpath); void load_hosts(); +void load_forwarding_map(); } // namespace util } // namespace gkfs diff --git a/include/config.hpp b/include/config.hpp index 982eddcf4..83b5f4bc2 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -24,6 +24,7 @@ namespace gkfs { namespace config { constexpr auto hostfile_path = "./gkfs_hosts.txt"; +constexpr auto forwarding_file_path = "./gkfs_forwarding.map"; namespace io { /* diff --git a/include/daemon/scheduler/agios.hpp b/include/daemon/scheduler/agios.hpp new file mode 100644 index 000000000..9ff1104f1 --- /dev/null +++ b/include/daemon/scheduler/agios.hpp @@ -0,0 +1,15 @@ +#ifndef IFS_SCHEDULER_HPP +#define IFS_SCHEDULER_HPP + +#include + +void agios_initialize(); +void agios_shutdown(); + +void *agios_callback(int64_t request_id); +void *agios_callback_aggregated(int64_t *requests, int32_t total); +void *agios_eventual_callback(int64_t request_id, void *info); + +unsigned long long int generate_unique_id(); + +#endif \ No newline at end of file diff --git a/include/global/rpc/distributor.hpp b/include/global/rpc/distributor.hpp index 2a0b79ff3..e2641d298 100644 --- a/include/global/rpc/distributor.hpp +++ b/include/global/rpc/distributor.hpp @@ -69,6 +69,23 @@ public: std::vector locate_directory_metadata(const std::string& path) const override; }; +class ForwarderDistributor : public Distributor { +private: + host_t fwd_host_; + unsigned int hosts_size_; + std::hash str_hash; +public: + ForwarderDistributor(host_t fwhost, unsigned int hosts_size); + + host_t localhost() const override; + + host_t locate_data(const std::string& path, const chunkid_t& chnk_id) const override; + + host_t locate_file_metadata(const std::string& path) const override; + + std::vector locate_directory_metadata(const std::string& path) const override; +}; + } // namespace rpc } // namespace gkfs diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 1a37a58a7..4332685de 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -13,21 +13,21 @@ VALID_DEP_OPTIONS="mogon2 mogon1 direct all" MOGON1_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" "verbs" + "syscall_intercept" "date" "verbs" "agios" ) MOGON2_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "syscall_intercept" "date" "agios" ) DIRECT_DEPS=( - "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" ) ALL_DEPS=( "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "syscall_intercept" "date" "agios" ) usage_short() { @@ -343,6 +343,16 @@ if check_dependency "ofi" "${DEP_CONFIG[@]}"; then fi fi +# AGIOS +if check_dependency "agios" "${DEP_CONFIG[@]}"; then + echo "############################################################ Installing: AGIOS" + CURR=${SOURCE}/agios + prepare_build_dir "${CURR}" + cd "${CURR}"/build + $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" .. + make install +fi + # Mercury if check_dependency "mercury" "${DEP_CONFIG[@]}"; then @@ -418,7 +428,7 @@ if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then CURR=${SOURCE}/syscall_intercept prepare_build_dir "${CURR}" cd "${CURR}"/build - $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF .. + $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL}" -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF .. make install fi diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index ca859984b..60fe48420 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -13,21 +13,21 @@ VALID_DEP_OPTIONS="mogon2 mogon1 direct all" MOGON1_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi-verbs" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "syscall_intercept" "date" "agios" ) MOGON2_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "syscall_intercept" "date" "agios" ) DIRECT_DEPS=( - "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" ) ALL_DEPS=( "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "syscall_intercept" "date" "agios" ) # Stop all backround jobs on interruption. @@ -360,6 +360,11 @@ if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then clonedeps "syscall_intercept" "https://github.com/pmem/syscall_intercept.git" "cc3412a2ad39f2e26cc307d5b155232811d7408e" "" "syscall_intercept.patch" & fi +# get AGIOS +if check_dependency "agios" "${DEP_CONFIG[@]}"; then +clonedeps "agios" "https://github.com/francielizanon/agios.git" "c26a6544200f823ebb8f890dd94e653d148bf226" "-b development" & +fi + # get date if check_dependency "date" "${DEP_CONFIG[@]}"; then clonedeps "date" "https://github.com/HowardHinnant/date.git" "e7e1482087f58913b80a20b04d5c58d9d6d90155" & diff --git a/src/client/preload.cpp b/src/client/preload.cpp index ad09cff32..e4374db76 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -36,6 +36,8 @@ namespace { // make sure that things are only initialized once pthread_once_t init_env_thread = PTHREAD_ONCE_INIT; +pthread_t mapper; +bool forwarding_running; inline void exit_error_msg(int errcode, const string& msg) { @@ -98,9 +100,22 @@ void init_ld_environment_() { } /* Setup distributor */ + #ifdef GKFS_ENABLE_FORWARDING + try { + gkfs::util::load_forwarding_map(); + + LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + } catch (std::exception& e){ + exit_error_msg(EXIT_FAILURE, fmt::format("Unable set the forwarding host '{}'", e.what())); + } + + auto forwarder_dist = std::make_shared(CTX->fwd_host_id(), CTX->hosts().size()); + CTX->distributor(forwarder_dist); + #else auto simple_hash_dist = std::make_shared(CTX->local_host_id(), CTX->hosts().size()); CTX->distributor(simple_hash_dist); + #endif LOG(INFO, "Retrieving file system configuration..."); @@ -111,6 +126,35 @@ void init_ld_environment_() { LOG(INFO, "Environment initialization successful."); } +#ifdef GKFS_ENABLE_FORWARDING +void *forwarding_mapper(void *p) { + while (forwarding_running) { + try { + gkfs::util::load_forwarding_map(); + + LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + } catch (std::exception& e){ + exit_error_msg(EXIT_FAILURE, fmt::format("Unable set the forwarding host '{}'", e.what())); + } + + // Sleeps for 10 seconds + sleep(10); + } +} +#endif + +void init_forwarding_mapper() { + forwarding_running = true; + + pthread_create(&mapper, NULL, forwarding_mapper, NULL); +} + +void destroy_forwarding_mapper() { + forwarding_running = false; + + pthread_join(mapper, NULL); +} + void log_prog_name() { std::string line; std::ifstream cmdline("/proc/self/cmdline"); @@ -170,6 +214,8 @@ void init_preload() { CTX->unprotect_user_fds(); + init_forwarding_mapper(); + gkfs::preload::start_interception(); } @@ -177,6 +223,7 @@ void init_preload() { * Called last when preload library is used with the LD_PRELOAD environment variable */ void destroy_preload() { + destroy_forwarding_mapper(); CTX->clear_hosts(); LOG(DEBUG, "Peer information deleted"); diff --git a/src/client/preload_context.cpp b/src/client/preload_context.cpp index ce696e74e..9b9f3eca6 100644 --- a/src/client/preload_context.cpp +++ b/src/client/preload_context.cpp @@ -121,6 +121,13 @@ void PreloadContext::local_host_id(uint64_t id) { local_host_id_ = id; } +uint64_t PreloadContext::fwd_host_id() const { + return fwd_host_id_; +} + +void PreloadContext::fwd_host_id(uint64_t id) { + fwd_host_id_ = id; +} RelativizeStatus PreloadContext::relativize_fd_path(int dirfd, const char* raw_path, std::string& relative_path, @@ -360,4 +367,4 @@ PreloadContext::unprotect_user_fds() { } } // namespace preload -} // namespace gkfs \ No newline at end of file +} // namespace gkfs diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index 4db07f20e..16d956e9f 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -176,6 +176,72 @@ vector> load_hostfile(const std::string& lfpath) { return hosts; } +#ifdef GKFS_ENABLE_FORWARDING +map load_forwarding_map_file(const std::string& lfpath) { + + LOG(DEBUG, "Loading forwarding map file file: \"{}\"", lfpath); + + ifstream lf(lfpath); + if (!lf) { + throw runtime_error(fmt::format("Failed to open forwarding map file '{}': {}", + lfpath, strerror(errno))); + } + map forwarding_map; + const regex line_re("^(\\S+)\\s+(\\S+)$", + regex::ECMAScript | regex::optimize); + string line; + string host; + uint64_t forwarder; + std::smatch match; + while (getline(lf, line)) { + if (!regex_match(line, match, line_re)) { + + LOG(ERROR, "Unrecognized line format: [path: '{}', line: '{}']", + lfpath, line); + + throw runtime_error( + fmt::format("unrecognized line format: '{}'", line)); + } + host = match[1]; + forwarder = std::stoi(match[2].str()); + forwarding_map[host] = forwarder; + } + return forwarding_map; +} +#endif + +#ifdef GKFS_ENABLE_FORWARDING +void load_forwarding_map() { + string forwarding_map_file; + + forwarding_map_file = gkfs::env::get_var(gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + + map forwarding_map; + + while (forwarding_map.size() == 0) { + try { + forwarding_map = load_forwarding_map_file(forwarding_map_file); + } catch (const exception& e) { + auto emsg = fmt::format("Failed to load forwarding map file: {}", e.what()); + throw runtime_error(emsg); + } + } + + //if (forwarding_map.size() == 0) { + // throw runtime_error(fmt::format("Forwarding map file is empty: '{}'", forwarding_map_file)); + //} + + auto local_hostname = get_my_hostname(true); + + if (forwarding_map.find(local_hostname) == forwarding_map.end()) { + throw runtime_error(fmt::format("Unable to determine the forwarder for host: '{}'", local_hostname)); + } + LOG(INFO, "Forwarding map loaded for '{}' as '{}'", local_hostname, forwarding_map[local_hostname]); + + CTX->fwd_host_id(forwarding_map[local_hostname]); +} +#endif + void load_hosts() { string hostfile; @@ -239,4 +305,4 @@ void load_hosts() { } } // namespace util -} // namespace gkfs \ No newline at end of file +} // namespace gkfs diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index e4ba89177..4df524b9d 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -12,6 +12,7 @@ set(DAEMON_SRC handler/srv_metadata.cpp handler/srv_data.cpp handler/srv_management.cpp + scheduler/agios.cpp ) set(DAEMON_HEADERS ../../include/config.hpp @@ -23,6 +24,7 @@ set(DAEMON_HEADERS ../../include/global/path_util.hpp ../../include/daemon/daemon.hpp ../../include/daemon/util.hpp + ../../include/daemon/scheduler/agios.hpp ../../include/daemon/ops/metadentry.hpp ../../include/daemon/classes/fs_data.hpp ../../include/daemon/classes/rpc_data.hpp @@ -40,6 +42,7 @@ target_link_libraries(gkfs_daemon env_util spdlog fmt::fmt + ${AGIOS_LIBRARIES} # margo libs ${ABT_LIBRARIES} mercury @@ -53,6 +56,7 @@ target_link_libraries(gkfs_daemon target_include_directories(gkfs_daemon PRIVATE + ${AGIOS_INCLUDE_DIRS} ${ABT_INCLUDE_DIRS} ${MARGO_INCLUDE_DIRS} ) diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 5a709f0e3..187aa2926 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -24,6 +24,9 @@ #include #include #include +#ifdef GKFS_ENABLE_AGIOS +#include +#endif #include #include @@ -209,6 +212,22 @@ void init_environment() { } GKFS_DATA->spdlogger()->info("Startup successful. Daemon is ready."); } +#ifdef GKFS_ENABLE_AGIOS +/** + * Initialize the AGIOS scheduling library + */ +void agios_initialize() { + char configuration[] = "/tmp/agios.conf"; + + if (!agios_init(NULL, NULL, configuration, 0)) { + GKFS_DATA->spdlogger()->error("{}() Failed to initialize AGIOS scheduler: '{}'", __func__, configuration); + + agios_exit(); + + throw; + } +} +#endif /** * Destroys the margo, argobots, and mercury environments diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index 553a9eaa5..2c3226abb 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -21,6 +21,13 @@ #include #include +#ifdef GKFS_ENABLE_AGIOS +#include + +#define AGIOS_READ 0 +#define AGIOS_WRITE 1 +#define AGIOS_SERVER_ID_IGNORE 0 +#endif using namespace std; @@ -142,6 +149,36 @@ static hg_return_t rpc_srv_write(hg_handle_t handle) { auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug("{}() path: {}, size: {}, offset: {}", __func__, in.path, bulk_size, in.offset); + #ifdef GKFS_ENABLE_AGIOS + int *data; + ABT_eventual eventual = ABT_EVENTUAL_NULL; + + /* creating eventual */ + ABT_eventual_create(sizeof(int64_t), &eventual); + + unsigned long long int request_id = generate_unique_id(); + char *agios_path = (char*) in.path; + + // We should call AGIOS before chunking (as that is an internal way to handle the requests) + if (!agios_add_request(agios_path, AGIOS_WRITE, in.offset, in.total_chunk_size, request_id, AGIOS_SERVER_ID_IGNORE, agios_eventual_callback, eventual)) { + GKFS_DATA->spdlogger()->error("{}() Failed to send request to AGIOS", __func__); + } else { + GKFS_DATA->spdlogger()->debug("{}() request {} was sent to AGIOS", __func__, request_id); + } + + /* Block until the eventual is signaled */ + ABT_eventual_wait(eventual, (void **)&data); + + unsigned long long int result = *data; + GKFS_DATA->spdlogger()->debug("{}() request {} was unblocked (offset = {})!", __func__, result, in.offset); + + ABT_eventual_free(&eventual); + + // Let AGIOS knows it can release the request, as it is completed + if (!agios_release_request(agios_path, AGIOS_WRITE, in.total_chunk_size, in.offset)) { + GKFS_DATA->spdlogger()->error("{}() Failed to release request from AGIOS", __func__); + } + #endif /* * 2. Set up buffers for pull bulk transfers */ @@ -199,8 +236,10 @@ static hg_return_t rpc_srv_write(hg_handle_t handle) { // Start to look for a chunk that hashes to this host with the first chunk in the buffer for (auto chnk_id_file = in.chunk_start; chnk_id_file < in.chunk_end || chnk_id_curr < in.chunk_n; chnk_id_file++) { // Continue if chunk does not hash to this host + #ifndef GKFS_ENABLE_FORWARDING if (distributor.locate_data(in.path, chnk_id_file) != host_id) continue; + #endif chnk_ids_host[chnk_id_curr] = chnk_id_file; // save this id to host chunk list // offset case. Only relevant in the first iteration of the loop and if the chunk hashes to this host if (chnk_id_file == in.chunk_start && in.offset > 0) { @@ -347,6 +386,36 @@ static hg_return_t rpc_srv_read(hg_handle_t handle) { auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug("{}() path: {}, size: {}, offset: {}", __func__, in.path, bulk_size, in.offset); + #ifdef GKFS_ENABLE_AGIOS + int *data; + ABT_eventual eventual = ABT_EVENTUAL_NULL; + + /* creating eventual */ + ABT_eventual_create(sizeof(int64_t), &eventual); + + unsigned long long int request_id = generate_unique_id(); + char *agios_path = (char*) in.path; + + // We should call AGIOS before chunking (as that is an internal way to handle the requests) + if (!agios_add_request(agios_path, AGIOS_READ, in.offset, in.total_chunk_size, request_id, AGIOS_SERVER_ID_IGNORE, agios_eventual_callback, eventual)) { + GKFS_DATA->spdlogger()->error("{}() Failed to send request to AGIOS", __func__); + } else { + GKFS_DATA->spdlogger()->debug("{}() request {} was sent to AGIOS", __func__, request_id); + } + + /* block until the eventual is signaled */ + ABT_eventual_wait(eventual, (void **)&data); + + unsigned long long int result = *data; + GKFS_DATA->spdlogger()->debug("{}() request {} was unblocked (offset = {})!", __func__, result, in.offset); + + ABT_eventual_free(&eventual); + + // let AGIOS knows it can release the request, as it is completed + if (!agios_release_request(agios_path, AGIOS_READ, in.total_chunk_size, in.offset)) { + GKFS_DATA->spdlogger()->error("{}() Failed to release request from AGIOS", __func__); + } + #endif /* * 2. Set up buffers for pull bulk transfers @@ -367,9 +436,11 @@ static hg_return_t rpc_srv_read(hg_handle_t handle) { GKFS_DATA->spdlogger()->error("{}() Failed to access allocated buffer from bulk handle", __func__); return gkfs::rpc::cleanup_respond(&handle, &in, &out, &bulk_handle); } + #ifndef GKFS_ENABLE_FORWARDING auto const host_id = in.host_id; auto const host_size = in.host_size; gkfs::rpc::SimpleHashDistributor distributor(host_id, host_size); + #endif auto path = make_shared(in.path); // chnk_ids used by this host @@ -397,8 +468,10 @@ static hg_return_t rpc_srv_read(hg_handle_t handle) { // Start to look for a chunk that hashes to this host with the first chunk in the buffer for (auto chnk_id_file = in.chunk_start; chnk_id_file < in.chunk_end || chnk_id_curr < in.chunk_n; chnk_id_file++) { // Continue if chunk does not hash to this host + #ifndef GKFS_ENABLE_FORWARDING if (distributor.locate_data(in.path, chnk_id_file) != host_id) continue; + #endif chnk_ids_host[chnk_id_curr] = chnk_id_file; // save this id to host chunk list // Only relevant in the first iteration of the loop and if the chunk hashes to this host if (chnk_id_file == in.chunk_start && in.offset > 0) { @@ -570,4 +643,13 @@ static hg_return_t rpc_srv_get_chunk_stat(hg_handle_t handle) { DEFINE_MARGO_RPC_HANDLER(rpc_srv_get_chunk_stat) +#ifdef GKFS_ENABLE_AGIOS +void *agios_eventual_callback(int64_t request_id, void *info) { + GKFS_DATA->spdlogger()->debug("{}() custom callback request {} is ready", __func__, request_id); + + ABT_eventual_set((ABT_eventual) info, &request_id, sizeof(int64_t)); + + return 0; +} +#endif diff --git a/src/daemon/scheduler/agios.cpp b/src/daemon/scheduler/agios.cpp new file mode 100644 index 000000000..2648c1c93 --- /dev/null +++ b/src/daemon/scheduler/agios.cpp @@ -0,0 +1,10 @@ +#include + +unsigned long long int generate_unique_id() { + // Calculates the hash of this request + timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + unsigned long long int id = ts.tv_sec*1000000000L + ts.tv_nsec; + + return id; +} \ No newline at end of file diff --git a/src/global/rpc/distributor.cpp b/src/global/rpc/distributor.cpp index ab0597853..cbb2f428f 100644 --- a/src/global/rpc/distributor.cpp +++ b/src/global/rpc/distributor.cpp @@ -68,5 +68,30 @@ locate_directory_metadata(const string& path) const { return {localhost_}; } +ForwarderDistributor:: +ForwarderDistributor(host_t fwhost, unsigned int hosts_size) : + fwd_host_(fwhost), + hosts_size_(hosts_size) +{} + +host_t ForwarderDistributor:: +localhost() const { + return fwd_host_; +} + +host_t ForwarderDistributor:: +locate_data(const std::string& path, const chunkid_t& chnk_id) const { + return fwd_host_; +} + +host_t ForwarderDistributor:: +locate_file_metadata(const std::string& path) const { + return str_hash(path) % hosts_size_; +} + +std::vector ForwarderDistributor:: +locate_directory_metadata(const std::string& path) const { + return {fwd_host_}; +} } // namespace rpc -} // namespace gkfs \ No newline at end of file +} // namespace gkfs -- GitLab From c0eebe715e374a0d8696ba8d99eb97f94682e94e Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 11:34:50 -0300 Subject: [PATCH 25/71] Restructure CI for GekkoFWD --- .gitlab-ci.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b1ac1266..42960550e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,9 +2,7 @@ stages: - download deps - build deps - build - - build forwarding - test - - test forwarding variables: DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" @@ -187,8 +185,8 @@ test lseek: paths: - "${LOG_PATH}" -test wr: - stage: test forwarding +test FWD wr: + stage: test dependencies: - "compile GekkoFWD" script: @@ -203,8 +201,8 @@ test wr: paths: - "${LOG_PATH}" -test directories: - stage: test forwarding +test FWD directories: + stage: test dependencies: - "compile GekkoFWD" script: @@ -219,8 +217,8 @@ test directories: paths: - "${LOG_PATH}" -test truncate: - stage: test forwarding +test FWD truncate: + stage: test dependencies: - "compile GekkoFWD" script: @@ -235,8 +233,8 @@ test truncate: paths: - "${LOG_PATH}" -test path resolution: - stage: test forwarding +test FWD path resolution: + stage: test dependencies: - "compile GekkoFWD" script: @@ -251,8 +249,8 @@ test path resolution: paths: - "${LOG_PATH}" -test lseek: - stage: test forwarding +test FWD lseek: + stage: test dependencies: - "compile GekkoFWD" script: -- GitLab From 7e59dd0c1762c719c7f05466114315330403b524 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 13:58:16 -0300 Subject: [PATCH 26/71] Define functions only for the forwarding mode --- src/client/preload.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/client/preload.cpp b/src/client/preload.cpp index e4374db76..14ec65f5c 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -36,8 +36,10 @@ namespace { // make sure that things are only initialized once pthread_once_t init_env_thread = PTHREAD_ONCE_INIT; +#ifdef GKFS_ENABLE_FORWARDING pthread_t mapper; bool forwarding_running; +#endif inline void exit_error_msg(int errcode, const string& msg) { @@ -143,17 +145,21 @@ void *forwarding_mapper(void *p) { } #endif +#ifdef GKFS_ENABLE_FORWARDING void init_forwarding_mapper() { forwarding_running = true; pthread_create(&mapper, NULL, forwarding_mapper, NULL); } +#endif +#ifdef GKFS_ENABLE_FORWARDING void destroy_forwarding_mapper() { forwarding_running = false; pthread_join(mapper, NULL); } +#endif void log_prog_name() { std::string line; @@ -214,7 +220,9 @@ void init_preload() { CTX->unprotect_user_fds(); + #ifdef GKFS_ENABLE_FORWARDING init_forwarding_mapper(); + #endif gkfs::preload::start_interception(); } @@ -223,7 +231,9 @@ void init_preload() { * Called last when preload library is used with the LD_PRELOAD environment variable */ void destroy_preload() { + #ifdef GKFS_ENABLE_FORWARDING destroy_forwarding_mapper(); + #endif CTX->clear_hosts(); LOG(DEBUG, "Peer information deleted"); -- GitLab From a98417ec835e15b4f1b90c1fb63df0b795267486 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 14:12:34 -0300 Subject: [PATCH 27/71] Update .gitlab-ci.yml --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 42960550e..e815f15e7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,10 +19,10 @@ variables: INTEGRATION_TESTS_FWD_BIN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration" INTEGRATION_TESTS_FWD_RUN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/run" - TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/gkfs/build" + TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build" PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" - TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test/gkfwd/build" + TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test/build" PYTEST_FWD: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" LOG_PATH: "${CI_PROJECT_DIR}/logs" @@ -92,8 +92,8 @@ compile GekkoFWD: -DGKFS_BUILD_TESTS:BOOL=ON -DGKFS_INSTALL_TESTS:BOOL=ON -DRPC_PROTOCOL="ofi+sockets" - -DENABLE_FORWARDING:BOOL=ON \ - -DENABLE_AGIOS:BOOL=ON \ + -DENABLE_FORWARDING:BOOL=ON + -DENABLE_AGIOS:BOOL=ON -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} -DCMAKE_INSTALL_PREFIX=${INSTALL_FWD_PATH} ${CI_PROJECT_DIR} -- GitLab From b4ed249fa67cc15204d69685c565a5f665bf9737 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 14:25:59 -0300 Subject: [PATCH 28/71] Working to fix integration tests --- .gitlab-ci.yml | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e815f15e7..c80560ecc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ stages: - build deps - build - test + - test forwarding variables: DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" @@ -102,7 +103,7 @@ compile GekkoFWD: paths: - ${INSTALL_FWD_PATH} -compile tests: +tests GekkoFS: stage: build dependencies: - "compile dependencies" @@ -114,6 +115,18 @@ compile tests: paths: - ${TESTS_BUILD_PATH} +tests GekkoFWD: + stage: build + dependencies: + - "compile dependencies" + script: + - mkdir -p ${TESTS_BUILD_FWD_PATH} && cd ${TESTS_BUILD_FWD_PATH} + - cmake -DCMAKE_BUILD_TYPE=Debug .. + - make -j$(nproc) + artifacts: + paths: + - ${TESTS_BUILD_FWD_PATH} + integration tests: stage: test script: @@ -186,7 +199,7 @@ test lseek: - "${LOG_PATH}" test FWD wr: - stage: test + stage: test forwarding dependencies: - "compile GekkoFWD" script: @@ -202,7 +215,7 @@ test FWD wr: - "${LOG_PATH}" test FWD directories: - stage: test + stage: test forwarding dependencies: - "compile GekkoFWD" script: @@ -218,7 +231,7 @@ test FWD directories: - "${LOG_PATH}" test FWD truncate: - stage: test + stage: test forwarding dependencies: - "compile GekkoFWD" script: @@ -234,7 +247,7 @@ test FWD truncate: - "${LOG_PATH}" test FWD path resolution: - stage: test + stage: test forwarding dependencies: - "compile GekkoFWD" script: @@ -250,7 +263,7 @@ test FWD path resolution: - "${LOG_PATH}" test FWD lseek: - stage: test + stage: test forwarding dependencies: - "compile GekkoFWD" script: -- GitLab From 4a79192d2dc5eed28c5d743840cb2ad2a8200714 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 15:06:06 -0300 Subject: [PATCH 29/71] Handle CI cache between branches --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c80560ecc..64b1f510a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,7 +43,7 @@ image: jeanbez/gekkofs-forwarding:latest compile dependencies: stage: build deps cache: - key: deps-cache + key: ${CI_COMMIT_REF_SLUG} paths: - ${DEPS_INSTALL_PATH}/ script: -- GitLab From e91b379212a6cbeb3a37212968f04972dff6b345 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 15:23:34 -0300 Subject: [PATCH 30/71] Try to fix missing library in CI --- .gitlab-ci.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 64b1f510a..7c5b264ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,6 @@ stages: - build deps - build - test - - test forwarding variables: DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" @@ -215,7 +214,7 @@ test FWD wr: - "${LOG_PATH}" test FWD directories: - stage: test forwarding + stage: test dependencies: - "compile GekkoFWD" script: @@ -231,7 +230,7 @@ test FWD directories: - "${LOG_PATH}" test FWD truncate: - stage: test forwarding + stage: test dependencies: - "compile GekkoFWD" script: @@ -247,7 +246,7 @@ test FWD truncate: - "${LOG_PATH}" test FWD path resolution: - stage: test forwarding + stage: test dependencies: - "compile GekkoFWD" script: @@ -263,7 +262,7 @@ test FWD path resolution: - "${LOG_PATH}" test FWD lseek: - stage: test forwarding + stage: test dependencies: - "compile GekkoFWD" script: -- GitLab From 73a6aa2b5b5ed5ead5e396a9d2b33e991f89261c Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 15:28:20 -0300 Subject: [PATCH 31/71] Try to fix missing library in CI --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7c5b264ee..2f4d1a112 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -198,7 +198,7 @@ test lseek: - "${LOG_PATH}" test FWD wr: - stage: test forwarding + stage: test dependencies: - "compile GekkoFWD" script: -- GitLab From 0ecca7ab8d46b6892def231e5d77e013c5e987ac Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 16:14:36 -0300 Subject: [PATCH 32/71] Try to fix missing library in CI --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2f4d1a112..27f921d10 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,10 +19,10 @@ variables: INTEGRATION_TESTS_FWD_BIN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration" INTEGRATION_TESTS_FWD_RUN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/run" - TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build" + TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test-gkfs/build" PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" - TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test/build" + TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test-gkfwd/build" PYTEST_FWD: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" LOG_PATH: "${CI_PROJECT_DIR}/logs" -- GitLab From eaa86bdd7b2e442454ab7cd79779c00fcc3c04d1 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 17:52:03 -0300 Subject: [PATCH 33/71] Try to fix missing library in CI --- .gitlab-ci.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27f921d10..cb1a10377 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,10 +19,10 @@ variables: INTEGRATION_TESTS_FWD_BIN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration" INTEGRATION_TESTS_FWD_RUN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/run" - TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test-gkfs/build" + TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/gkfs-test/build" PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" - TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test-gkfwd/build" + TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/gkfwd-test/build" PYTEST_FWD: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" LOG_PATH: "${CI_PROJECT_DIR}/logs" @@ -42,7 +42,7 @@ image: jeanbez/gekkofs-forwarding:latest compile dependencies: stage: build deps cache: - key: ${CI_COMMIT_REF_SLUG} + key: deps-build paths: - ${DEPS_INSTALL_PATH}/ script: @@ -201,6 +201,7 @@ test FWD wr: stage: test dependencies: - "compile GekkoFWD" + - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - cp agios/* /tmp @@ -217,6 +218,7 @@ test FWD directories: stage: test dependencies: - "compile GekkoFWD" + - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - cp agios/* /tmp @@ -233,6 +235,7 @@ test FWD truncate: stage: test dependencies: - "compile GekkoFWD" + - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - cp agios/* /tmp @@ -249,6 +252,7 @@ test FWD path resolution: stage: test dependencies: - "compile GekkoFWD" + - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - cp agios/* /tmp @@ -265,6 +269,7 @@ test FWD lseek: stage: test dependencies: - "compile GekkoFWD" + - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - cp agios/* /tmp -- GitLab From c1644e2a91cb44216217252a59efe06ea1b1d4b5 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 00:01:52 -0300 Subject: [PATCH 34/71] Working to fix CI issue --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cb1a10377..08b2c21eb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,10 +19,10 @@ variables: INTEGRATION_TESTS_FWD_BIN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration" INTEGRATION_TESTS_FWD_RUN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/run" - TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/gkfs-test/build" + TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build-gkfs" PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" - TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/gkfwd-test/build" + TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test/build-gkfwd" PYTEST_FWD: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" LOG_PATH: "${CI_PROJECT_DIR}/logs" -- GitLab From 10ca0da7144000e7a1cffd911c4929e7d7c669fc Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 00:17:06 -0300 Subject: [PATCH 35/71] Working to fix CI issue --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 08b2c21eb..7bba1c91d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -200,6 +200,7 @@ test lseek: test FWD wr: stage: test dependencies: + - "compile dependencies" - "compile GekkoFWD" - "tests GekkoFWD" script: -- GitLab From 036d095813228b57e2fec9808099e076ac50d8cf Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 00:46:43 -0300 Subject: [PATCH 36/71] Include dependency on CI tests --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7bba1c91d..ad5de816f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -218,6 +218,7 @@ test FWD wr: test FWD directories: stage: test dependencies: + - "compile dependencies" - "compile GekkoFWD" - "tests GekkoFWD" script: @@ -235,6 +236,7 @@ test FWD directories: test FWD truncate: stage: test dependencies: + - "compile dependencies" - "compile GekkoFWD" - "tests GekkoFWD" script: @@ -252,6 +254,7 @@ test FWD truncate: test FWD path resolution: stage: test dependencies: + - "compile dependencies" - "compile GekkoFWD" - "tests GekkoFWD" script: @@ -269,6 +272,7 @@ test FWD path resolution: test FWD lseek: stage: test dependencies: + - "compile dependencies" - "compile GekkoFWD" - "tests GekkoFWD" script: -- GitLab From 48ccef9e3dc94cc2cfa608ac48ccfc6eca886f4c Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 01:30:52 -0300 Subject: [PATCH 37/71] Temporarily remove AGIOS from tests --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad5de816f..d2d998a09 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,7 +93,6 @@ compile GekkoFWD: -DGKFS_INSTALL_TESTS:BOOL=ON -DRPC_PROTOCOL="ofi+sockets" -DENABLE_FORWARDING:BOOL=ON - -DENABLE_AGIOS:BOOL=ON -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} -DCMAKE_INSTALL_PREFIX=${INSTALL_FWD_PATH} ${CI_PROJECT_DIR} -- GitLab From a0eb3bc01ea406fa39acc2b5ea5e6c464eec606b Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 02:06:34 -0300 Subject: [PATCH 38/71] Enable AGIOS integration for GekkoFWD --- .gitlab-ci.yml | 1 + src/daemon/daemon.cpp | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d2d998a09..ad5de816f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,6 +93,7 @@ compile GekkoFWD: -DGKFS_INSTALL_TESTS:BOOL=ON -DRPC_PROTOCOL="ofi+sockets" -DENABLE_FORWARDING:BOOL=ON + -DENABLE_AGIOS:BOOL=ON -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} -DCMAKE_INSTALL_PREFIX=${INSTALL_FWD_PATH} ${CI_PROJECT_DIR} diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 187aa2926..59e9df6ef 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -162,6 +162,20 @@ void init_environment() { throw; } + #ifdef GKFS_ENABLE_FORWARDING + GKFS_DATA->spdlogger()->debug("{}() Enable I/O forwarding mode", __func__); + #endif + + #ifdef GKFS_ENABLE_AGIOS + // Initialize AGIOS scheduler + GKFS_DATA->spdlogger()->debug("{}() Initializing AGIOS scheduler: '{}'", __func__, "/tmp/agios.conf"); + try { + agios_initialize(); + } catch (const std::exception & e) { + GKFS_DATA->spdlogger()->error("{}() Failed to initialize AGIOS scheduler: {}", __func__, e.what()); + throw; + } + #endif // Initialize data backend std::string chunk_storage_path = GKFS_DATA->rootdir() + "/data/chunks"s; GKFS_DATA->spdlogger()->debug("{}() Initializing storage backend: '{}'", __func__, chunk_storage_path); -- GitLab From 2458f9cea6f4489d7196885067f095eee04ec830 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 14:21:56 -0300 Subject: [PATCH 39/71] Include forwarding test support in the new test harness --- CMakeLists.txt | 9 + src/client/preload.cpp | 6 +- tests/integration/conftest.py | 22 +- tests/integration/harness/cli.py | 8 + tests/integration/harness/gkfs.py | 435 ++++++++++++++++++++++++ tests/integration/pytest.ini.in | 2 +- tests/integration/pytest.install.ini.in | 2 +- 7 files changed, 477 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c515b50bd..8293bb122 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -232,6 +232,15 @@ if(GKFS_BUILD_TESTS) message(STATUS "[gekkofs] Preparing tests...") set(GKFS_TESTS_INTERFACE "lo" CACHE STRING "Network interface to use when running tests (default: lo)") message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}") + + message(STATUS "[gekkofs] Check for forwarding tests...") + if(ENABLE_FORWARDING) + set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)") + else() + set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)") + endif() + message(STATUS "[gekkofs] Forwarding tests: ${GKFS_TESTS_FORWARDING}") + add_subdirectory(tests) else() unset(GKFS_TESTS_INTERFACE CACHE) diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 14ec65f5c..4f878c1ca 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -135,13 +135,17 @@ void *forwarding_mapper(void *p) { gkfs::util::load_forwarding_map(); LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); - } catch (std::exception& e){ + } catch (std::exception& e) { exit_error_msg(EXIT_FAILURE, fmt::format("Unable set the forwarding host '{}'", e.what())); + + return; } // Sleeps for 10 seconds sleep(10); } + + return; } #endif diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 91d1ffe2a..6039ef0f7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -19,7 +19,7 @@ from pathlib import Path from harness.logger import logger, initialize_logging, finalize_logging from harness.cli import add_cli_options, set_default_log_formatter from harness.workspace import Workspace, FileCreator -from harness.gkfs import Daemon, Client, ShellClient +from harness.gkfs import Daemon, Client, ShellClient, FwdDaemon, FwdClient, ShellFwdClient from harness.reporter import report_test_status, report_test_headline, report_assertion_pass def pytest_configure(config): @@ -87,27 +87,41 @@ def gkfs_daemon(test_workspace, request): interface = request.config.getoption('--interface') - daemon = Daemon(interface, test_workspace) + if request.config.getoption('--forwarding') == 'ON': + daemon = FwdDaemon(interface, test_workspace) + else: + daemon = Daemon(interface, test_workspace) + yield daemon.run() daemon.shutdown() @pytest.fixture -def gkfs_client(test_workspace): +def gkfs_client(test_workspace, request): """ Sets up a gekkofs client environment so that operations (system calls, library calls, ...) can be requested from a co-running daemon. """ + interface = request.config.getoption('--interface') + + if request.config.getoption('--forwarding') == 'ON': + return FwdClient(test_workspace) + return Client(test_workspace) @pytest.fixture -def gkfs_shell(test_workspace): +def gkfs_shell(test_workspace, request): """ Sets up a gekkofs environment so that shell commands (stat, ls, mkdir, etc.) can be issued to a co-running daemon. """ + interface = request.config.getoption('--interface') + + if request.config.getoption('--forwarding') == 'ON': + return ShellFwdClient(test_workspace) + return ShellClient(test_workspace) @pytest.fixture diff --git a/tests/integration/harness/cli.py b/tests/integration/harness/cli.py index 3144c9b8c..bd2b10966 100644 --- a/tests/integration/harness/cli.py +++ b/tests/integration/harness/cli.py @@ -33,6 +33,14 @@ def add_cli_options(parser): help="network interface used for communications (default: 'lo')." ) + parser.addoption( + '--forwarding', + action='store', + type=str, + default='ON', + help="enable the forwarding mode (default: 'OFF')." + ) + parser.addoption( "--bin-dir", action='append', diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index 36abe40ec..738b51b6a 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -26,6 +26,7 @@ gkfs_daemon_cmd = 'gkfs_daemon' gkfs_client_cmd = 'gkfs.io' gkfs_client_lib_file = 'libgkfs_intercept.so' gkfs_hosts_file = 'gkfs_hosts.txt' +gkfs_forwarding_map_file = 'gkfs_forwarding.map' gkfs_daemon_log_file = 'gkfs_daemon.log' gkfs_daemon_log_level = '100' gkfs_client_log_file = 'gkfs_client.log' @@ -582,3 +583,437 @@ class ShellClient: @property def cwd(self): return self._workspace.twd + +class FwdDaemon: + def __init__(self, interface, workspace): + + self._address = get_ephemeral_address(interface) + self._workspace = workspace + + self._cmd = sh.Command(gkfs_daemon_cmd, self._workspace.bindirs) + self._env = os.environ.copy() + + libdirs = ':'.join( + filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + + [str(p) for p in self._workspace.libdirs])) + + self._patched_env = { + 'LD_LIBRARY_PATH' : libdirs, + 'GKFS_HOSTS_FILE' : self.cwd / gkfs_hosts_file, + 'GKFS_DAEMON_LOG_PATH' : self.logdir / gkfs_daemon_log_file, + 'GKFS_LOG_LEVEL' : gkfs_daemon_log_level + } + self._env.update(self._patched_env) + + def run(self): + + args = [ '--mountdir', self.mountdir, + '--rootdir', self.rootdir, + '-l', self._address ] + + logger.debug(f"spawning daemon") + logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, args))) + logger.debug(f"patched env:\n{pformat(self._patched_env)}") + + self._proc = self._cmd( + args, + _env=self._env, +# _out=sys.stdout, +# _err=sys.stderr, + _bg=True, + ) + + logger.debug(f"daemon process spawned (PID={self._proc.pid})") + logger.debug("waiting for daemon to be ready") + + try: + self.wait_until_active(self._proc.pid, 10.0) + except Exception as ex: + logger.error(f"daemon initialization failed: {ex}") + + # if the daemon initialized correctly but took longer than + # `timeout`, we may be leaving running processes behind + if _process_exists(self._proc.pid): + self.shutdown() + + logger.critical(f"daemon was shut down, what is ex? {ex.__repr__}?") + raise ex + + logger.debug("daemon is ready") + + return self + + def wait_until_active(self, pid, timeout, max_lines=50): + """ + Waits until a GKFS daemon is active or until a certain timeout + has expired. Checks if the daemon is running by searching its + log for a pre-defined readiness message. + + Parameters + ---------- + pid: `int` + The PID of the daemon process to wait for. + + timeout: `number` + The number of seconds to wait for + + max_lines: `int` + The maximum number of log lines to check for a match. + """ + + gkfs_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' + + init_time = perf_counter() + + while perf_counter() - init_time < timeout: + try: + logger.debug(f"checking log file") + with open(self.logdir / gkfs_daemon_log_file) as log: + for line in islice(log, max_lines): + if re.search(gkfs_daemon_active_log_pattern, line) is not None: + return + except FileNotFoundError: + # Log is missing, the daemon might have crashed... + logger.debug(f"daemon log file missing, checking if daemon is alive...") + + pid=self._proc.pid + + if not _process_exists(pid): + raise RuntimeError(f"process {pid} is not running") + + # ... or it might just be lazy. let's give it some more time + logger.debug(f"daemon {pid} found, retrying...") + + raise RuntimeError("initialization timeout exceeded") + + def shutdown(self): + logger.debug(f"terminating daemon") + + try: + self._proc.terminate() + err = self._proc.wait() + except sh.SignalException_SIGTERM: + pass + except Exception: + raise + + + @property + def cwd(self): + return self._workspace.twd + + @property + def rootdir(self): + return self._workspace.rootdir + + @property + def mountdir(self): + return self._workspace.mountdir + + @property + def logdir(self): + return self._workspace.logdir + + @property + def interface(self): + return self._interface + +class FwdClient: + """ + A class to represent a GekkoFS client process with a patched LD_PRELOAD. + This class allows tests to interact with the file system using I/O-related + function calls, be them system calls (e.g. read()) or glibc I/O functions + (e.g. opendir()). + """ + def __init__(self, workspace): + self._parser = IOParser() + self._workspace = workspace + self._cmd = sh.Command(gkfs_client_cmd, self._workspace.bindirs) + self._env = os.environ.copy() + + # create the forwarding map file + fwd_map_file = open(self.cwd / gkfs_forwarding_map_file, 'w') + fwd_map_file.write('{} {}\n'.format(socket.gethostname(), 0)) + fwd_map_file.close() + + libdirs = ':'.join( + filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + + [str(p) for p in self._workspace.libdirs])) + + # ensure the client interception library is available: + # to avoid running code with potentially installed libraries, + # it must be found in one (and only one) of the workspace's bindirs + preloads = [] + for d in self._workspace.bindirs: + search_path = Path(d) / gkfs_client_lib_file + if search_path.exists(): + preloads.append(search_path) + + if len(preloads) == 0: + logger.error(f'No client libraries found in the test\'s binary directories:') + pytest.exit("Aborted due to initialization error. Check test logs.") + + if len(preloads) != 1: + logger.error(f'Multiple client libraries found in the test\'s binary directories:') + for p in preloads: + logger.error(f' {p}') + logger.error(f'Make sure that only one copy of the client library is available.') + pytest.exit("Aborted due to initialization error. Check test logs.") + + self._preload_library = preloads[0] + + self._patched_env = { + 'LD_LIBRARY_PATH' : libdirs, + 'LD_PRELOAD' : self._preload_library, + 'LIBGKFS_HOSTS_FILE' : self.cwd / gkfs_hosts_file, + 'LIBGKFS_FORWARDING_MAP_FILE' : self.cwd / gkfs_forwarding_map_file, + 'LIBGKFS_LOG' : gkfs_client_log_level, + 'LIBGKFS_LOG_OUTPUT' : self._workspace.logdir / gkfs_client_log_file + } + + self._env.update(self._patched_env) + + @property + def preload_library(self): + """ + Return the preload library detected for this client + """ + + return self._preload_library + + def run(self, cmd, *args): + + logger.debug(f"running client") + logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, list(args)))) + logger.debug(f"patched env: {pformat(self._patched_env)}") + + out = self._cmd( + [ cmd ] + list(args), + _env = self._env, + # _out=sys.stdout, + # _err=sys.stderr, + ) + + logger.debug(f"command output: {out.stdout}") + return self._parser.parse(cmd, out.stdout) + + def __getattr__(self, name): + return _proxy_exec(self, name) + + @property + def cwd(self): + return self._workspace.twd + +class ShellFwdClient: + """ + A class to represent a GekkoFS shell client process. + This class allows tests to execute shell commands or scripts via bash -c + on a GekkoFS instance. + """ + + def __init__(self, workspace): + self._workspace = workspace + self._cmd = sh.Command("bash") + self._env = os.environ.copy() + + # create the forwarding map file + fwd_map_file = open(self.cwd / gkfs_forwarding_map_file, 'w') + fwd_map_file.write('{} {}\n'.format(socket.gethostname(), 0)) + fwd_map_file.close() + + libdirs = ':'.join( + filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + + [str(p) for p in self._workspace.libdirs])) + + # ensure the client interception library is available: + # to avoid running code with potentially installed libraries, + # it must be found in one (and only one) of the workspace's bindirs + preloads = [] + for d in self._workspace.bindirs: + search_path = Path(d) / gkfs_client_lib_file + if search_path.exists(): + preloads.append(search_path) + + if len(preloads) != 1: + logger.error(f'Multiple client libraries found in the test\'s binary directories:') + for p in preloads: + logger.error(f' {p}') + logger.error(f'Make sure that only one copy of the client library is available.') + pytest.exit("Aborted due to initialization error") + + self._preload_library = preloads[0] + + self._patched_env = { + 'LD_LIBRARY_PATH' : libdirs, + 'LD_PRELOAD' : self._preload_library, + 'LIBGKFS_HOSTS_FILE' : self.cwd / gkfs_hosts_file, + 'LIBGKFS_FORWARDING_MAP_FILE' : self.cwd / gkfs_forwarding_map_file, + 'LIBGKFS_LOG' : gkfs_client_log_level, + 'LIBGKFS_LOG_OUTPUT' : self._workspace.logdir / gkfs_client_log_file + } + + self._env.update(self._patched_env) + + @property + def patched_environ(self): + """ + Return the patched environment required to run a test as a string that + can be prepended to a shell command. + """ + + return ' '.join(f'{k}="{v}"' for k,v in self._patched_env.items()) + + def script(self, code, intercept_shell=True, timeout=60, timeout_signal=signal.SIGKILL): + """ + Execute a shell script passed as an argument in bash. + + For instance, the following snippet: + + mountdir = pathlib.Path('/tmp') + file01 = 'file01' + + ShellClient().script( + f''' + expected_pathname={mountdir / file01} + if [[ -e ${{expected_pathname}} ]]; + then + exit 0 + fi + exit 1 + ''') + + transforms into: + + bash -c ' + expected_pathname=/tmp/file01 + if [[ -e ${expected_pathname} ]]; + then + exit 0 + fi + exit 1 + ' + + Note that since we are using Python's f-strings, for variable + expansions to work correctly, they need to be defined with double + braces, e.g. ${{expected_pathname}}. + + Parameters + ---------- + code: `str` + The script code to be passed to 'bash -c'. + + intercept_shell: `bool` + Controls whether the shell executing the script should be + executed with LD_PRELOAD=libgkfs_intercept.so (default: True). + + timeout: `int` + How much time, in seconds, we should give the process to complete. + If the process does not finish within the timeout, it will be sent + the signal defined by `timeout_signal`. + + Default value: 60 + + timeout_signal: `int` + The signal to be sent to the process if `timeout` is not None. + + Default value: signal.SIGKILL + + Returns + ------- + A sh.RunningCommand instance that allows interacting with + the finished process. + """ + + logger.debug(f"running bash") + logger.debug(f"cmd: bash -c '{code}'") + logger.debug(f"timeout: {timeout} seconds") + logger.debug(f"timeout_signal: {signal.Signals(timeout_signal).name}") + + if intercept_shell: + logger.debug(f"patched env: {self._patched_env}") + + # 'sh' raises an exception if the return code is not zero; + # since we'd rather check for return codes explictly, we + # whitelist all exit codes from 1 to 255 as 'ok' using the + # _ok_code argument + return self._cmd('-c', + code, + _env = (self._env if intercept_shell else os.environ), + # _out=sys.stdout, + # _err=sys.stderr, + _timeout=timeout, + _timeout_signal=timeout_signal, + # _ok_code=list(range(0, 256)) + ) + + def run(self, cmd, *args, timeout=60, timeout_signal=signal.SIGKILL): + """ + Execute a shell command with arguments. + + For example, the following snippet: + + mountdir = pathlib.Path('/tmp') + file01 = 'file01' + + ShellClient().stat('--terse', mountdir / file01) + + transforms into: + + bash -c 'stat --terse /tmp/file01' + + Parameters: + ----------- + cmd: `str` + The command to execute. + + args: `list` + The list of arguments for the command. + + timeout: `number` + How much time, in seconds, we should give the process to complete. + If the process does not finish within the timeout, it will be sent + the signal defined by `timeout_signal`. + + Default value: 60 + + timeout_signal: `int` + The signal to be sent to the process if `timeout` is not None. + + Default value: signal.SIGKILL + + Returns + ------- + A ShellCommand instance that allows interacting with the finished + process. Note that ShellCommand wraps sh.RunningCommand and adds s + extra properties to it. + """ + + bash_c_args = f"{cmd} {' '.join(str(a) for a in args)}" + logger.debug(f"running bash") + logger.debug(f"cmd: bash -c '{bash_c_args}'") + logger.debug(f"timeout: {timeout} seconds") + logger.debug(f"timeout_signal: {signal.Signals(timeout_signal).name}") + logger.debug(f"patched env:\n{pformat(self._patched_env)}") + + # 'sh' raises an exception if the return code is not zero; + # since we'd rather check for return codes explictly, we + # whitelist all exit codes from 1 to 255 as 'ok' using the + # _ok_code argument + proc = self._cmd('-c', + bash_c_args, + _env = self._env, + # _out=sys.stdout, + # _err=sys.stderr, + _timeout=timeout, + _timeout_signal=timeout_signal, + # _ok_code=list(range(0, 256)) + ) + + return ShellCommand(cmd, proc) + + def __getattr__(self, name): + return _proxy_exec(self, name) + + @property + def cwd(self): + return self._workspace.twd diff --git a/tests/integration/pytest.ini.in b/tests/integration/pytest.ini.in index d7901fa8f..3edd1784c 100644 --- a/tests/integration/pytest.ini.in +++ b/tests/integration/pytest.ini.in @@ -13,5 +13,5 @@ [pytest] testpaths = @CMAKE_CURRENT_SOURCE_DIR@ -addopts = @PYTEST_BINDIR_ARGS@ @PYTEST_LIBDIR_ARGS@ --interface=@GKFS_TESTS_INTERFACE@ +addopts = @PYTEST_BINDIR_ARGS@ @PYTEST_LIBDIR_ARGS@ --interface=@GKFS_TESTS_INTERFACE@ --forwarding=@GKFS_TESTS_FORWARDING@ enable_assertion_pass_hook = true diff --git a/tests/integration/pytest.install.ini.in b/tests/integration/pytest.install.ini.in index 23e87ec25..0d6889ef7 100644 --- a/tests/integration/pytest.install.ini.in +++ b/tests/integration/pytest.install.ini.in @@ -13,5 +13,5 @@ [pytest] testpaths = @CMAKE_INSTALL_FULL_DATAROOTDIR@/gkfs/tests -addopts = --bin-dir=@CMAKE_INSTALL_FULL_BINDIR@ --bin-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --lib-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --interface=@GKFS_TESTS_INTERFACE@ +addopts = --bin-dir=@CMAKE_INSTALL_FULL_BINDIR@ --bin-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --lib-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --interface=@GKFS_TESTS_FORWARDING@ --forwarding=@ENABLE_FORWARDING@ enable_assertion_pass_hook = true -- GitLab From dc3c8a2c5f309331c47b82f8af335b121fee09c8 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 14:34:05 -0300 Subject: [PATCH 40/71] Organize the CI and fix small bug --- .gitlab-ci.yml | 22 +++++++++++++++++----- src/client/preload.cpp | 4 ---- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad5de816f..97d22d45c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ stages: - build deps - build - test + - test forwarding variables: DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" @@ -137,6 +138,17 @@ integration tests: paths: - "${INTEGRATION_TESTS_RUN_PATH}" +integration GekkoFWD tests: + stage: test GekkoFWD + script: + - mkdir -p ${INTEGRATION_TESTS_FWD_RUN_PATH} + - cd ${INTEGRATION_TESTS_FWD_BIN_PATH} + - TMPDIR=${INTEGRATION_TESTS_FWD_RUN_PATH} unbuffer ${PYTEST} -v | tee ${INTEGRATION_TESTS_FWD_RUN_PATH}/session.log + artifacts: + when: on_failure + paths: + - "${INTEGRATION_TESTS_FWD_RUN_PATH}" + test wr: stage: test script: @@ -198,7 +210,7 @@ test lseek: - "${LOG_PATH}" test FWD wr: - stage: test + stage: test forwarding dependencies: - "compile dependencies" - "compile GekkoFWD" @@ -216,7 +228,7 @@ test FWD wr: - "${LOG_PATH}" test FWD directories: - stage: test + stage: test forwarding dependencies: - "compile dependencies" - "compile GekkoFWD" @@ -234,7 +246,7 @@ test FWD directories: - "${LOG_PATH}" test FWD truncate: - stage: test + stage: test forwarding dependencies: - "compile dependencies" - "compile GekkoFWD" @@ -252,7 +264,7 @@ test FWD truncate: - "${LOG_PATH}" test FWD path resolution: - stage: test + stage: test forwarding dependencies: - "compile dependencies" - "compile GekkoFWD" @@ -270,7 +282,7 @@ test FWD path resolution: - "${LOG_PATH}" test FWD lseek: - stage: test + stage: test forwarding dependencies: - "compile dependencies" - "compile GekkoFWD" diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 4f878c1ca..7d1cfd3bc 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -137,15 +137,11 @@ void *forwarding_mapper(void *p) { LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); } catch (std::exception& e) { exit_error_msg(EXIT_FAILURE, fmt::format("Unable set the forwarding host '{}'", e.what())); - - return; } // Sleeps for 10 seconds sleep(10); } - - return; } #endif -- GitLab From 427e7374fa5ccd73f4f9c4a47768f681c4f27c6e Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 14:36:01 -0300 Subject: [PATCH 41/71] Fix typo in CI .yaml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 97d22d45c..bf3594a52 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -139,7 +139,7 @@ integration tests: - "${INTEGRATION_TESTS_RUN_PATH}" integration GekkoFWD tests: - stage: test GekkoFWD + stage: test forwarding script: - mkdir -p ${INTEGRATION_TESTS_FWD_RUN_PATH} - cd ${INTEGRATION_TESTS_FWD_BIN_PATH} -- GitLab From 4d8f6fc37b3e32384dae54618adca123eca3631d Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 13 May 2020 18:31:10 -0300 Subject: [PATCH 42/71] Fix typo in variable name for the interface --- .gitlab-ci.yml | 2 +- tests/integration/pytest.install.ini.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf3594a52..6b52e0245 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -138,7 +138,7 @@ integration tests: paths: - "${INTEGRATION_TESTS_RUN_PATH}" -integration GekkoFWD tests: +integration FWD tests: stage: test forwarding script: - mkdir -p ${INTEGRATION_TESTS_FWD_RUN_PATH} diff --git a/tests/integration/pytest.install.ini.in b/tests/integration/pytest.install.ini.in index 0d6889ef7..a8668942b 100644 --- a/tests/integration/pytest.install.ini.in +++ b/tests/integration/pytest.install.ini.in @@ -13,5 +13,5 @@ [pytest] testpaths = @CMAKE_INSTALL_FULL_DATAROOTDIR@/gkfs/tests -addopts = --bin-dir=@CMAKE_INSTALL_FULL_BINDIR@ --bin-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --lib-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --interface=@GKFS_TESTS_FORWARDING@ --forwarding=@ENABLE_FORWARDING@ +addopts = --bin-dir=@CMAKE_INSTALL_FULL_BINDIR@ --bin-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --lib-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --interface=@GKFS_TESTS_INTERFACE@ --forwarding=@ENABLE_FORWARDING@ enable_assertion_pass_hook = true -- GitLab From dc1e0f3117128403c6b378ba3c17fcf347f1967e Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 19 May 2020 12:41:41 -0300 Subject: [PATCH 43/71] Remove sudo from Dockerfile --- docker/build_env.docker | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/build_env.docker b/docker/build_env.docker index ef83cbaa9..346807b86 100644 --- a/docker/build_env.docker +++ b/docker/build_env.docker @@ -80,7 +80,6 @@ RUN yum -y -q update && yum -y -q install \ automake \ cmake \ cmake3 \ - sudo \ gcc \ gcc-c++ \ openssl-devel \ -- GitLab From 8cd8081bfc07563f1345f56abf0efaca1daf7a16 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 19 May 2020 12:50:35 -0300 Subject: [PATCH 44/71] Fix typo --- src/client/gkfs_functions.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 29854a518..f87465139 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -456,7 +456,7 @@ ssize_t gkfs_write(int fd, const void* buf, size_t count) { ssize_t gkfs_pwritev(int fd, const struct iovec* iov, int iovcnt, off_t offset) { auto file = CTX->file_map()->get(fd); - auto pos = offset; // keep truck of current position + auto pos = offset; // keep track of current position ssize_t written = 0; ssize_t ret; for (int i = 0; i < iovcnt; ++i) { @@ -530,7 +530,7 @@ ssize_t gkfs_read(int fd, void* buf, size_t count) { ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) { auto file = CTX->file_map()->get(fd); - auto pos = offset; // keep truck of current position + auto pos = offset; // keep track of current position ssize_t read = 0; ssize_t ret; for (int i = 0; i < iovcnt; ++i) { -- GitLab From 10ad84f34510ab744c01dfaadafcb97d7a32a207 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 19 May 2020 12:53:22 -0300 Subject: [PATCH 45/71] Rename ENABLE_FORWARDING to GKFS_ENABLE_FORWARDING --- .gitlab-ci.yml | 4 ++-- CMakeLists.txt | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6b52e0245..4f60e8730 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,8 +93,8 @@ compile GekkoFWD: -DGKFS_BUILD_TESTS:BOOL=ON -DGKFS_INSTALL_TESTS:BOOL=ON -DRPC_PROTOCOL="ofi+sockets" - -DENABLE_FORWARDING:BOOL=ON - -DENABLE_AGIOS:BOOL=ON + -DGFKS_ENABLE_FORWARDING:BOOL=ON + -DGFKS_ENABLE_AGIOS:BOOL=ON -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} -DCMAKE_INSTALL_PREFIX=${INSTALL_FWD_PATH} ${CI_PROJECT_DIR} diff --git a/CMakeLists.txt b/CMakeLists.txt index 8293bb122..fcc8288ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,17 +142,17 @@ if(ENABLE_CLIENT_LOG) add_definitions(-DGKFS_ENABLE_LOGGING) endif() message(STATUS "[gekkofs] Client logging output: ${ENABLE_CLIENT_LOG}") -option(ENABLE_FORWARDING "Enable forwarding mode" OFF) -if(ENABLE_FORWARDING) +option(GKFS_ENABLE_FORWARDING "Enable forwarding mode" OFF) +if(GKFS_ENABLE_FORWARDING) add_definitions(-DGKFS_ENABLE_FORWARDING) endif() -message(STATUS "[gekkofs] Forwarding mode: ${ENABLE_FORWARDING}") +message(STATUS "[gekkofs] Forwarding mode: ${GKFS_ENABLE_FORWARDING}") -option(ENABLE_AGIOS "Enable AGIOS scheduling library" OFF) -if(ENABLE_AGIOS) +option(GKFS_ENABLE_AGIOS "Enable AGIOS scheduling library" OFF) +if(GKFS_ENABLE_AGIOS) add_definitions(-DGKFS_ENABLE_AGIOS) endif() -message(STATUS "[gekkofs] AGIOS scheduling: ${ENABLE_AGIOS}") +message(STATUS "[gekkofs] AGIOS scheduling: ${GKFS_ENABLE_AGIOS}") set(CLIENT_LOG_MESSAGE_SIZE 1024 CACHE STRING "Maximum size of a log message in the client library") add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${CLIENT_LOG_MESSAGE_SIZE}) -- GitLab From 6e4ab6a161accfef1756f63fef34208775cedc10 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 19 May 2020 12:53:42 -0300 Subject: [PATCH 46/71] Follow Gekko coding conventions --- src/client/preload.cpp | 2 +- src/daemon/handler/srv_data.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 7d1cfd3bc..9959254bc 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -129,7 +129,7 @@ void init_ld_environment_() { } #ifdef GKFS_ENABLE_FORWARDING -void *forwarding_mapper(void *p) { +void *forwarding_mapper(void* p) { while (forwarding_running) { try { gkfs::util::load_forwarding_map(); diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index 2c3226abb..4eae87c9b 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -644,11 +644,11 @@ static hg_return_t rpc_srv_get_chunk_stat(hg_handle_t handle) { DEFINE_MARGO_RPC_HANDLER(rpc_srv_get_chunk_stat) #ifdef GKFS_ENABLE_AGIOS -void *agios_eventual_callback(int64_t request_id, void *info) { +void *agios_eventual_callback(int64_t request_id, void* info) { GKFS_DATA->spdlogger()->debug("{}() custom callback request {} is ready", __func__, request_id); ABT_eventual_set((ABT_eventual) info, &request_id, sizeof(int64_t)); - + return 0; } #endif -- GitLab From 3f6ebd98535dea136d12cd2d7d3bd4e481f61408 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 19 May 2020 16:32:58 -0300 Subject: [PATCH 47/71] Create configuration directory --- {agios => config/GekkoFWD/agios}/access_times.func | 0 {agios => config/GekkoFWD/agios}/agios.conf | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {agios => config/GekkoFWD/agios}/access_times.func (100%) rename {agios => config/GekkoFWD/agios}/agios.conf (100%) diff --git a/agios/access_times.func b/config/GekkoFWD/agios/access_times.func similarity index 100% rename from agios/access_times.func rename to config/GekkoFWD/agios/access_times.func diff --git a/agios/agios.conf b/config/GekkoFWD/agios/agios.conf similarity index 100% rename from agios/agios.conf rename to config/GekkoFWD/agios/agios.conf -- GitLab From ecdfa12ab031e8f893438639524391b87fb19392 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 19 May 2020 20:08:38 -0300 Subject: [PATCH 48/71] Fix CI path with AGIOS configuration file --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4f60e8730..5efd5759f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -217,7 +217,7 @@ test FWD wr: - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - - cp agios/* /tmp + - cp config/GekkoFWD/agios/* /tmp - echo "`hostname` 0" > gkfs_forwarding.map - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - sleep 4 @@ -235,7 +235,7 @@ test FWD directories: - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - - cp agios/* /tmp + - cp config/GekkoFWD/agios/* /tmp - echo "`hostname` 0" > gkfs_forwarding.map - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - sleep 4 @@ -253,7 +253,7 @@ test FWD truncate: - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - - cp agios/* /tmp + - cp config/GekkoFWD/agios/* /tmp - echo "`hostname` 0" > gkfs_forwarding.map - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - sleep 4 @@ -271,7 +271,7 @@ test FWD path resolution: - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - - cp agios/* /tmp + - cp config/GekkoFWD/agios/* /tmp - echo "`hostname` 0" > gkfs_forwarding.map - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - sleep 4 @@ -289,7 +289,7 @@ test FWD lseek: - "tests GekkoFWD" script: - mkdir -p "${LOG_PATH}" - - cp agios/* /tmp + - cp config/GekkoFWD/agios/* /tmp - echo "`hostname` 0" > gkfs_forwarding.map - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - sleep 4 -- GitLab From a1a3a04d09dc9fb4e0bd9c4982a20eff2a78e4f2 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 25 May 2020 10:23:31 -0300 Subject: [PATCH 49/71] Create a target for GekkoFWD and merge the fix of statx to test --- CMakeLists.txt | 22 +++-- include/client/gkfs_functions.hpp | 2 + include/client/hooks.hpp | 2 + include/global/rpc/distributor.hpp | 4 +- src/client/CMakeLists.txt | 96 +++++++++++++++++++ src/client/gkfs_functions.cpp | 3 +- src/client/hooks.cpp | 6 +- src/client/intercept.cpp | 2 +- src/client/preload.cpp | 2 + src/daemon/CMakeLists.txt | 86 ++++++++++++++++- tests/integration/CMakeLists.txt | 16 ++-- tests/integration/forwarding/test_map.py | 60 ++++++++++++ .../integration/harness/gkfs.io/commands.hpp | 2 + tests/integration/harness/gkfs.io/main.cpp | 2 + .../integration/harness/gkfs.io/serialize.hpp | 6 +- tests/integration/harness/gkfs.io/statx.cpp | 3 +- tests/integration/harness/io.py | 2 +- 17 files changed, 285 insertions(+), 31 deletions(-) create mode 100644 tests/integration/forwarding/test_map.py diff --git a/CMakeLists.txt b/CMakeLists.txt index fcc8288ed..c4c1af288 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,17 +142,9 @@ if(ENABLE_CLIENT_LOG) add_definitions(-DGKFS_ENABLE_LOGGING) endif() message(STATUS "[gekkofs] Client logging output: ${ENABLE_CLIENT_LOG}") -option(GKFS_ENABLE_FORWARDING "Enable forwarding mode" OFF) -if(GKFS_ENABLE_FORWARDING) - add_definitions(-DGKFS_ENABLE_FORWARDING) -endif() -message(STATUS "[gekkofs] Forwarding mode: ${GKFS_ENABLE_FORWARDING}") +option(GKFS_ENABLE_FORWARDING "Enable forwarding mode" OFF) option(GKFS_ENABLE_AGIOS "Enable AGIOS scheduling library" OFF) -if(GKFS_ENABLE_AGIOS) - add_definitions(-DGKFS_ENABLE_AGIOS) -endif() -message(STATUS "[gekkofs] AGIOS scheduling: ${GKFS_ENABLE_AGIOS}") set(CLIENT_LOG_MESSAGE_SIZE 1024 CACHE STRING "Maximum size of a log message in the client library") add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${CLIENT_LOG_MESSAGE_SIZE}) @@ -215,6 +207,18 @@ include_directories( ) include(GNUInstallDirs) +include(CheckSymbolExists) + +check_cxx_source_compiles(" + #include + #include + + int main() { + struct statx buf; + statx(AT_FDCWD, \"/foo\", AT_EMPTY_PATH, STATX_BASIC_STATS, &buf); + return 0; + } +" GLIBC_HAS_STATX) # Global components add_subdirectory(src/global) diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index bda2fc57a..86ec30448 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -41,7 +41,9 @@ int gkfs_stat(const std::string& path, struct stat* buf, bool follow_links = tru // Implementation of statx, it uses the normal stat and maps the information to the statx structure // Follow links is true by default +#ifdef STATX_TYPE int gkfs_statx(int dirfd, const std::string& path, int flags, unsigned int mask,struct statx* buf, bool follow_links = true ); +#endif int gkfs_statfs(struct statfs* buf); diff --git a/include/client/hooks.hpp b/include/client/hooks.hpp index 2f2a65463..9af366dd4 100644 --- a/include/client/hooks.hpp +++ b/include/client/hooks.hpp @@ -30,7 +30,9 @@ int hook_close(int fd); int hook_stat(const char* path, struct stat* buf); +#ifdef STATX_TYPE int hook_statx(int dirfd, const char* path, int flags, unsigned int mask,struct statx* buf); +#endif int hook_lstat(const char* path, struct stat* buf); diff --git a/include/global/rpc/distributor.hpp b/include/global/rpc/distributor.hpp index e2641d298..86b85e6fa 100644 --- a/include/global/rpc/distributor.hpp +++ b/include/global/rpc/distributor.hpp @@ -77,9 +77,9 @@ private: public: ForwarderDistributor(host_t fwhost, unsigned int hosts_size); - host_t localhost() const override; + host_t localhost() const override final; - host_t locate_data(const std::string& path, const chunkid_t& chnk_id) const override; + host_t locate_data(const std::string& path, const chunkid_t& chnk_id) const override final; host_t locate_file_metadata(const std::string& path) const override; diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 0be20f8ea..44429c726 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -79,3 +79,99 @@ install(TARGETS gkfs_intercept PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) +if(GKFS_ENABLE_FORWARDING) + set(FWD_PRELOAD_SRC + gkfs_functions.cpp + hooks.cpp + intercept.cpp + logging.cpp + open_file_map.cpp + open_dir.cpp + path.cpp + preload.cpp + preload_context.cpp + preload_util.cpp + ../global/path_util.cpp + ../global/rpc/rpc_util.cpp + rpc/rpc_types.cpp + rpc/forward_data.cpp + rpc/forward_management.cpp + rpc/forward_metadata.cpp + syscalls/detail/syscall_info.c + ) + set(FWD_PRELOAD_HEADERS + ../../include/client/gkfs_functions.hpp + ../../include/config.hpp + ../../include/client/env.hpp + ../../include/client/hooks.hpp + ../../include/client/intercept.hpp + ../../include/client/logging.hpp + ../../include/client/make_array.hpp + ../../include/client/open_file_map.hpp + ../../include/client/open_dir.hpp + ../../include/client/path.hpp + ../../include/client/preload.hpp + ../../include/client/preload_context.hpp + ../../include/client/preload_util.hpp + ../../include/client/rpc/rpc_types.hpp + ../../include/client/rpc/forward_management.hpp + ../../include/client/rpc/forward_metadata.hpp + ../../include/client/rpc/forward_data.hpp + ../../include/client/syscalls/args.hpp + ../../include/client/syscalls/decoder.hpp + ../../include/client/syscalls/errno.hpp + ../../include/client/syscalls/rets.hpp + ../../include/client/syscalls/syscall.hpp + ../../include/client/syscalls/detail/syscall_info.h + ../../include/global/cmake_configure.hpp + ../../include/global/chunk_calc_util.hpp + ../../include/global/global_defs.hpp + ../../include/global/path_util.hpp + ../../include/global/rpc/rpc_types.hpp + ../../include/global/rpc/rpc_util.hpp + ) + + add_library(gkfwd_intercept SHARED ${FWD_PRELOAD_SRC} ${FWD_PRELOAD_HEADERS}) + + if(GKFS_ENABLE_FORWARDING) + set_target_properties(gkfwd_intercept + PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING + ) + endif() + message(STATUS "[gekkofs] Forwarding mode: ${GKFS_ENABLE_FORWARDING}") + + if(GKFS_ENABLE_AGIOS) + set_target_properties(gkfwd_intercept + PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_AGIOS + ) + endif() + message(STATUS "[gekkofs] AGIOS scheduling: ${GKFS_ENABLE_AGIOS}") + + target_link_libraries(gkfwd_intercept + # internal + metadata + distributor + env_util + # external + Syscall_intercept::Syscall_intercept + dl + mercury + hermes + fmt::fmt + Boost::boost # needed for tokenizer header + Threads::Threads + Date::TZ + ) + + target_include_directories(gkfwd_intercept + PRIVATE + ${ABT_INCLUDE_DIRS} + ${MARGO_INCLUDE_DIRS} + ) + + install(TARGETS gkfwd_intercept + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs + ) +endif() \ No newline at end of file diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index f87465139..a4c5507bc 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -232,7 +232,7 @@ int gkfs_stat(const string& path, struct stat* buf, bool follow_links) { return 0; } - +#ifdef STATX_TYPE int gkfs_statx(int dirfs, const std::string& path, int flags, unsigned int mask, struct statx* buf, bool follow_links) { auto md = gkfs::util::get_metadata(path, follow_links); if (!md) { @@ -268,6 +268,7 @@ int gkfs_statx(int dirfs, const std::string& path, int flags, unsigned int mask, return 0; } +#endif int gkfs_statfs(struct statfs* buf) { auto blk_stat = gkfs::rpc::forward_get_chunk_stat(); diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index b84f6f1d7..7ca89a4fe 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -98,7 +98,7 @@ int hook_stat(const char* path, struct stat* buf) { return syscall_no_intercept(SYS_stat, rel_path.c_str(), buf); } - +#ifdef STATX_TYPE int hook_statx(int dirfd, const char* path, int flags, unsigned int mask, struct ::statx* buf) { LOG(DEBUG, "{}() called with dirfd: '{}', path: \"{}\", flags: '{}', mask: '{}', buf: '{}'", @@ -127,9 +127,7 @@ int hook_statx(int dirfd, const char* path, int flags, unsigned int mask, struct return syscall_no_intercept(SYS_statx, dirfd, path, flags, mask, buf); } - - - +#endif int hook_lstat(const char* path, struct stat* buf) { diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index fe1a766e0..708cafe6d 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -466,7 +466,7 @@ int hook(long syscall_number, reinterpret_cast(arg1)); break; -#ifdef SYS_statx +#ifdef STATX_TYPE case SYS_statx: *result = gkfs::hook::hook_statx(static_cast(arg0), reinterpret_cast(arg1), diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 9959254bc..e687bf488 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -142,6 +142,8 @@ void *forwarding_mapper(void* p) { // Sleeps for 10 seconds sleep(10); } + + return nullptr; } #endif diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 4df524b9d..e63b0bd00 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -12,7 +12,6 @@ set(DAEMON_SRC handler/srv_metadata.cpp handler/srv_data.cpp handler/srv_management.cpp - scheduler/agios.cpp ) set(DAEMON_HEADERS ../../include/config.hpp @@ -24,7 +23,6 @@ set(DAEMON_HEADERS ../../include/global/path_util.hpp ../../include/daemon/daemon.hpp ../../include/daemon/util.hpp - ../../include/daemon/scheduler/agios.hpp ../../include/daemon/ops/metadentry.hpp ../../include/daemon/classes/fs_data.hpp ../../include/daemon/classes/rpc_data.hpp @@ -42,7 +40,6 @@ target_link_libraries(gkfs_daemon env_util spdlog fmt::fmt - ${AGIOS_LIBRARIES} # margo libs ${ABT_LIBRARIES} mercury @@ -56,7 +53,6 @@ target_link_libraries(gkfs_daemon target_include_directories(gkfs_daemon PRIVATE - ${AGIOS_INCLUDE_DIRS} ${ABT_INCLUDE_DIRS} ${MARGO_INCLUDE_DIRS} ) @@ -64,3 +60,85 @@ target_include_directories(gkfs_daemon install(TARGETS gkfs_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) + + +if(GKFS_ENABLE_FORWARDING) + set(FWD_DAEMON_SRC + ../global/rpc/rpc_util.cpp + ../global/path_util.cpp + daemon.cpp + util.cpp + ops/metadentry.cpp + classes/fs_data.cpp + classes/rpc_data.cpp + handler/srv_metadata.cpp + handler/srv_data.cpp + handler/srv_management.cpp + scheduler/agios.cpp + ) + set(FWD_DAEMON_HEADERS + ../../include/config.hpp + ../../include/version.hpp + ../../include/global/cmake_configure.hpp + ../../include/global/global_defs.hpp + ../../include/global/rpc/rpc_types.hpp + ../../include/global/rpc/rpc_util.hpp + ../../include/global/path_util.hpp + ../../include/daemon/daemon.hpp + ../../include/daemon/util.hpp + ../../include/daemon/scheduler/agios.hpp + ../../include/daemon/ops/metadentry.hpp + ../../include/daemon/classes/fs_data.hpp + ../../include/daemon/classes/rpc_data.hpp + ../../include/daemon/handler/rpc_defs.hpp + ../../include/daemon/handler/rpc_util.hpp + ) + add_executable(gkfwd_daemon ${FWD_DAEMON_SRC} ${FWD_DAEMON_HEADERS}) + + if(GKFS_ENABLE_FORWARDING) + set_target_properties(gkfwd_daemon + PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING + ) + endif() + message(STATUS "[gekkofs] Forwarding mode: ${GKFS_ENABLE_FORWARDING}") + + if(GKFS_ENABLE_AGIOS) + set_target_properties(gkfwd_daemon + PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_AGIOS + ) + endif() + message(STATUS "[gekkofs] AGIOS scheduling: ${GKFS_ENABLE_AGIOS}") + + target_link_libraries(gkfwd_daemon + # internal libs + metadata + metadata_db + storage + distributor + log_util + env_util + spdlog + fmt::fmt + ${AGIOS_LIBRARIES} + # margo libs + ${ABT_LIBRARIES} + mercury + ${MARGO_LIBRARIES} + # others + Boost::boost # needed for tokenizer header + Boost::program_options + Boost::filesystem + Threads::Threads + ) + + target_include_directories(gkfwd_daemon + PRIVATE + ${AGIOS_INCLUDE_DIRS} + ${ABT_INCLUDE_DIRS} + ${MARGO_INCLUDE_DIRS} + ) + + install(TARGETS gkfwd_daemon + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + ) +endif() \ No newline at end of file diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 3ba045901..fe550aebe 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -16,12 +16,14 @@ gkfs_add_python_test( SOURCE directories/test_directories.py ) -gkfs_add_python_test( - NAME test_status - PYTHON_VERSION 3.6 - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration - SOURCE status/test_status.py -) +if (GLIBC_HAS_STATX) + gkfs_add_python_test( + NAME test_status + PYTHON_VERSION 3.6 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration + SOURCE status/test_status.py + ) +endif() gkfs_add_python_test( NAME test_operations @@ -30,12 +32,14 @@ gkfs_add_python_test( SOURCE operations/ ) +if (GLIBC_HAS_STATX) gkfs_add_python_test( NAME test_lseek PYTHON_VERSION 3.6 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration SOURCE position/test_lseek.py ) +endif() gkfs_add_python_test( NAME test_shell diff --git a/tests/integration/forwarding/test_map.py b/tests/integration/forwarding/test_map.py new file mode 100644 index 000000000..7a52dd997 --- /dev/null +++ b/tests/integration/forwarding/test_map.py @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# SPDX-License-Identifier: MIT # +################################################################################ + +import harness +from pathlib import Path +import errno +import stat +import os +import ctypes +import sh +import sys +import pytest +from harness.logger import logger + +nonexisting = "nonexisting" + + +def test_read(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + # create a file in gekkofs + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf = b'42' + ret = gkfs_client.write(file, buf, len(buf)) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = gkfs_client.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + ret = gkfs_client.read(file, len(buf)) + + assert ret.buf == buf + assert ret.retval == len(buf) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index edda1c289..7d111c609 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -59,8 +59,10 @@ writev_init(CLI::App& app); void pwritev_init(CLI::App& app); +#ifdef STATX_TYPE void statx_init(CLI::App& app); +#endif void lseek_init(CLI::App& app); diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 0cde949db..e946d1b11 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -34,7 +34,9 @@ init_commands(CLI::App& app) { pwrite_init(app); writev_init(app); pwritev_init(app); + #ifdef STATX_TYPE statx_init(app); + #endif lseek_init(app); } diff --git a/tests/integration/harness/gkfs.io/serialize.hpp b/tests/integration/harness/gkfs.io/serialize.hpp index ceb22db80..df1a7b1db 100644 --- a/tests/integration/harness/gkfs.io/serialize.hpp +++ b/tests/integration/harness/gkfs.io/serialize.hpp @@ -105,6 +105,7 @@ struct adl_serializer { } }; +#ifdef STATX_TYPE // ADL specialization for struct ::statx_timestamp type template <> struct adl_serializer { @@ -116,8 +117,7 @@ struct adl_serializer { }; } }; - - +#endif // ADL specialization for struct ::dirent type template <> @@ -162,6 +162,7 @@ struct adl_serializer { } }; +#ifdef STATX_TYPE // ADL specialization for struct ::statx type template <> struct adl_serializer { @@ -192,6 +193,7 @@ struct adl_serializer { }; } }; +#endif } // namespace nlohmann diff --git a/tests/integration/harness/gkfs.io/statx.cpp b/tests/integration/harness/gkfs.io/statx.cpp index 0ffe481ba..a50a39f85 100644 --- a/tests/integration/harness/gkfs.io/statx.cpp +++ b/tests/integration/harness/gkfs.io/statx.cpp @@ -32,6 +32,7 @@ using json = nlohmann::json; unsigned int mask, struct statx *statxbuf); */ +#ifdef STATX_TYPE struct statx_options { bool verbose; int dirfd; @@ -135,4 +136,4 @@ statx_init(CLI::App& app) { statx_exec(*opts); }); } - +#endif \ No newline at end of file diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index f50d3d9f4..521dae757 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -318,4 +318,4 @@ class IOParser: if command in self.OutputSchemas: return self.OutputSchemas[command].loads(output) else: - raise ValueError(f"Unknown I/O command {cmd}") + raise ValueError(f"Unknown I/O command {command}") -- GitLab From 9893f57e2168336dffd7d14f4e6692a09fe2a021 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 25 May 2020 23:16:21 -0300 Subject: [PATCH 50/71] Include I/O forwarding tests --- tests/integration/CMakeLists.txt | 16 +++++- tests/integration/conftest.py | 34 ++++++++---- tests/integration/forwarding/test_map.py | 66 +++++++++++++++++++++--- tests/integration/harness/cli.py | 8 --- tests/integration/harness/gkfs.py | 59 +++++++++++++-------- 5 files changed, 135 insertions(+), 48 deletions(-) diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index fe550aebe..5d050cee8 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -48,6 +48,13 @@ gkfs_add_python_test( SOURCE shell/ ) +gkfs_add_python_test( + NAME forwarding + PYTHON_VERSION 3.6 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration + SOURCE forwarding/ +) + if(GKFS_INSTALL_TESTS) install(DIRECTORY harness DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration @@ -90,7 +97,6 @@ if(GKFS_INSTALL_TESTS) PATTERN ".pytest_cache" EXCLUDE ) - install(DIRECTORY shell DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration FILES_MATCHING @@ -98,4 +104,12 @@ if(GKFS_INSTALL_TESTS) PATTERN "__pycache__" EXCLUDE PATTERN ".pytest_cache" EXCLUDE ) + + install(DIRECTORY forwarding + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration + FILES_MATCHING + REGEX ".*\\.py" + PATTERN "__pycache__" EXCLUDE + PATTERN ".pytest_cache" EXCLUDE + ) endif() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6039ef0f7..232bf209c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -20,6 +20,7 @@ from harness.logger import logger, initialize_logging, finalize_logging from harness.cli import add_cli_options, set_default_log_formatter from harness.workspace import Workspace, FileCreator from harness.gkfs import Daemon, Client, ShellClient, FwdDaemon, FwdClient, ShellFwdClient +from harness.factory import FwdDaemonCreator, FwdClientCreator from harness.reporter import report_test_status, report_test_headline, report_assertion_pass def pytest_configure(config): @@ -86,12 +87,8 @@ def gkfs_daemon(test_workspace, request): """ interface = request.config.getoption('--interface') + daemon = Daemon(interface, test_workspace) - if request.config.getoption('--forwarding') == 'ON': - daemon = FwdDaemon(interface, test_workspace) - else: - daemon = Daemon(interface, test_workspace) - yield daemon.run() daemon.shutdown() @@ -105,9 +102,6 @@ def gkfs_client(test_workspace, request): interface = request.config.getoption('--interface') - if request.config.getoption('--forwarding') == 'ON': - return FwdClient(test_workspace) - return Client(test_workspace) @pytest.fixture @@ -119,9 +113,6 @@ def gkfs_shell(test_workspace, request): interface = request.config.getoption('--interface') - if request.config.getoption('--forwarding') == 'ON': - return ShellFwdClient(test_workspace) - return ShellClient(test_workspace) @pytest.fixture @@ -132,3 +123,24 @@ def file_factory(test_workspace): """ return FileCreator(test_workspace) + +@pytest.fixture +def gkfwd_daemon_factory(test_workspace, request): + """ + Returns a factory that can create forwarding daemons + in the test workspace. + """ + + interface = request.config.getoption('--interface') + + return FwdDaemonCreator(interface, test_workspace) + +@pytest.fixture +def gkfwd_client_factory(test_workspace, request): + """ + Sets up a gekkofs client environment so that + operations (system calls, library calls, ...) can + be requested from a co-running daemon. + """ + + return FwdClientCreator(test_workspace) diff --git a/tests/integration/forwarding/test_map.py b/tests/integration/forwarding/test_map.py index 7a52dd997..41d0507a1 100644 --- a/tests/integration/forwarding/test_map.py +++ b/tests/integration/forwarding/test_map.py @@ -16,6 +16,7 @@ from pathlib import Path import errno import stat import os +import time import ctypes import sh import sys @@ -25,12 +26,19 @@ from harness.logger import logger nonexisting = "nonexisting" -def test_read(gkfs_daemon, gkfs_client): +def test_write_two_io_nodes(gkfwd_daemon_factory, gkfwd_client_factory): + """Write files from two clients using two daemons""" - file = gkfs_daemon.mountdir / "file" + d00 = gkfwd_daemon_factory.create() + d01 = gkfwd_daemon_factory.create() + + c00 = gkfwd_client_factory.create('c-0') + c01 = gkfwd_client_factory.create('c-1') + + file = d00.mountdir / "file-c00" # create a file in gekkofs - ret = gkfs_client.open(file, + ret = c00.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) @@ -39,13 +47,13 @@ def test_read(gkfs_daemon, gkfs_client): # write a buffer we know buf = b'42' - ret = gkfs_client.write(file, buf, len(buf)) + ret = c00.write(file, buf, len(buf)) assert ret.retval == len(buf) # Return the number of written bytes assert ret.errno == 115 #FIXME: Should be 0! # open the file to read - ret = gkfs_client.open(file, + ret = c00.open(file, os.O_RDONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) @@ -53,8 +61,54 @@ def test_read(gkfs_daemon, gkfs_client): assert ret.errno == 115 #FIXME: Should be 0! # read the file - ret = gkfs_client.read(file, len(buf)) + ret = c00.read(file, len(buf)) assert ret.buf == buf assert ret.retval == len(buf) # Return the number of read bytes assert ret.errno == 115 #FIXME: Should be 0! + + + file = d01.mountdir / "file-c01" + + # create a file in gekkofs + ret = c01.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf = b'42' + ret = c01.write(file, buf, len(buf)) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = c01.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + ret = c01.read(file, len(buf)) + + assert ret.buf == buf + assert ret.retval == len(buf) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # both files should be there and accessible by the two clients + ret = c00.readdir(d00.mountdir) + + assert len(ret.dirents) == 2 + + assert ret.dirents[0].d_name == 'file-c00' + assert ret.dirents[0].d_type == 8 # DT_REG + assert ret.errno == 115 #FIXME: Should be 0! + + assert ret.dirents[1].d_name == 'file-c01' + assert ret.dirents[1].d_type == 8 # DT_REG + assert ret.errno == 115 #FIXME: Should be 0! \ No newline at end of file diff --git a/tests/integration/harness/cli.py b/tests/integration/harness/cli.py index bd2b10966..3144c9b8c 100644 --- a/tests/integration/harness/cli.py +++ b/tests/integration/harness/cli.py @@ -33,14 +33,6 @@ def add_cli_options(parser): help="network interface used for communications (default: 'lo')." ) - parser.addoption( - '--forwarding', - action='store', - type=str, - default='ON', - help="enable the forwarding mode (default: 'OFF')." - ) - parser.addoption( "--bin-dir", action='append', diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index 738b51b6a..72308298d 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -26,13 +26,24 @@ gkfs_daemon_cmd = 'gkfs_daemon' gkfs_client_cmd = 'gkfs.io' gkfs_client_lib_file = 'libgkfs_intercept.so' gkfs_hosts_file = 'gkfs_hosts.txt' -gkfs_forwarding_map_file = 'gkfs_forwarding.map' gkfs_daemon_log_file = 'gkfs_daemon.log' gkfs_daemon_log_level = '100' gkfs_client_log_file = 'gkfs_client.log' gkfs_client_log_level = 'all' gkfs_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' +gkfwd_daemon_cmd = 'gkfwd_daemon' +gkfwd_client_cmd = 'gkfs.io' +gkfwd_client_lib_file = 'libgkfwd_intercept.so' +gkfwd_hosts_file = 'gkfs_hosts.txt' +gkfwd_forwarding_map_file = 'gkfs_forwarding.map' +gkfwd_daemon_log_file = 'gkfwd_daemon.log' +gkfwd_daemon_log_level = '100' +gkfwd_client_log_file = 'gkfwd_client.log' +gkfwd_client_log_level = 'all' +gkfwd_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' + + def get_ip_addr(iface): return netifaces.ifaddresses(iface)[netifaces.AF_INET][0]['addr'] @@ -590,7 +601,7 @@ class FwdDaemon: self._address = get_ephemeral_address(interface) self._workspace = workspace - self._cmd = sh.Command(gkfs_daemon_cmd, self._workspace.bindirs) + self._cmd = sh.Command(gkfwd_daemon_cmd, self._workspace.bindirs) self._env = os.environ.copy() libdirs = ':'.join( @@ -599,9 +610,9 @@ class FwdDaemon: self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, - 'GKFS_HOSTS_FILE' : self.cwd / gkfs_hosts_file, - 'GKFS_DAEMON_LOG_PATH' : self.logdir / gkfs_daemon_log_file, - 'GKFS_LOG_LEVEL' : gkfs_daemon_log_level + 'GKFS_HOSTS_FILE' : self.cwd / gkfwd_hosts_file, + 'GKFS_DAEMON_LOG_PATH' : self.logdir / gkfwd_daemon_log_file, + 'GKFS_LOG_LEVEL' : gkfwd_daemon_log_level } self._env.update(self._patched_env) @@ -668,9 +679,9 @@ class FwdDaemon: while perf_counter() - init_time < timeout: try: logger.debug(f"checking log file") - with open(self.logdir / gkfs_daemon_log_file) as log: + with open(self.logdir / gkfwd_daemon_log_file) as log: for line in islice(log, max_lines): - if re.search(gkfs_daemon_active_log_pattern, line) is not None: + if re.search(gkfwd_daemon_active_log_pattern, line) is not None: return except FileNotFoundError: # Log is missing, the daemon might have crashed... @@ -718,6 +729,7 @@ class FwdDaemon: def interface(self): return self._interface + class FwdClient: """ A class to represent a GekkoFS client process with a patched LD_PRELOAD. @@ -725,15 +737,18 @@ class FwdClient: function calls, be them system calls (e.g. read()) or glibc I/O functions (e.g. opendir()). """ - def __init__(self, workspace): + def __init__(self, workspace, identifier): self._parser = IOParser() self._workspace = workspace - self._cmd = sh.Command(gkfs_client_cmd, self._workspace.bindirs) + self._identifier = identifier + self._cmd = sh.Command(gkfwd_client_cmd, self._workspace.bindirs) self._env = os.environ.copy() + gkfwd_forwarding_map_file_local = '{}-{}'.format(identifier, gkfwd_forwarding_map_file) + # create the forwarding map file - fwd_map_file = open(self.cwd / gkfs_forwarding_map_file, 'w') - fwd_map_file.write('{} {}\n'.format(socket.gethostname(), 0)) + fwd_map_file = open(self.cwd / gkfwd_forwarding_map_file_local, 'w') + fwd_map_file.write('{} {}\n'.format(socket.gethostname(), int(identifier.split('-')[1]))) fwd_map_file.close() libdirs = ':'.join( @@ -745,7 +760,7 @@ class FwdClient: # it must be found in one (and only one) of the workspace's bindirs preloads = [] for d in self._workspace.bindirs: - search_path = Path(d) / gkfs_client_lib_file + search_path = Path(d) / gkfwd_client_lib_file if search_path.exists(): preloads.append(search_path) @@ -765,10 +780,10 @@ class FwdClient: self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, 'LD_PRELOAD' : self._preload_library, - 'LIBGKFS_HOSTS_FILE' : self.cwd / gkfs_hosts_file, - 'LIBGKFS_FORWARDING_MAP_FILE' : self.cwd / gkfs_forwarding_map_file, + 'LIBGKFS_HOSTS_FILE' : self.cwd / gkfwd_hosts_file, + 'LIBGKFS_FORWARDING_MAP_FILE' : self.cwd / gkfwd_forwarding_map_file_local, 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : self._workspace.logdir / gkfs_client_log_file + 'LIBGKFS_LOG_OUTPUT' : self._workspace.logdir / gkfwd_client_log_file } self._env.update(self._patched_env) @@ -817,7 +832,7 @@ class ShellFwdClient: self._env = os.environ.copy() # create the forwarding map file - fwd_map_file = open(self.cwd / gkfs_forwarding_map_file, 'w') + fwd_map_file = open(self.cwd / gkfwd_forwarding_map_file, 'w') fwd_map_file.write('{} {}\n'.format(socket.gethostname(), 0)) fwd_map_file.close() @@ -830,7 +845,7 @@ class ShellFwdClient: # it must be found in one (and only one) of the workspace's bindirs preloads = [] for d in self._workspace.bindirs: - search_path = Path(d) / gkfs_client_lib_file + search_path = Path(d) / gkfwd_client_lib_file if search_path.exists(): preloads.append(search_path) @@ -846,10 +861,10 @@ class ShellFwdClient: self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, 'LD_PRELOAD' : self._preload_library, - 'LIBGKFS_HOSTS_FILE' : self.cwd / gkfs_hosts_file, - 'LIBGKFS_FORWARDING_MAP_FILE' : self.cwd / gkfs_forwarding_map_file, - 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : self._workspace.logdir / gkfs_client_log_file + 'LIBGKFS_HOSTS_FILE' : self.cwd / gkfwd_hosts_file, + 'LIBGKFS_FORWARDING_MAP_FILE' : self.cwd / gkfwd_forwarding_map_file, + 'LIBGKFS_LOG' : gkfwd_client_log_level, + 'LIBGKFS_LOG_OUTPUT' : self._workspace.logdir / gkfwd_client_log_file } self._env.update(self._patched_env) @@ -904,7 +919,7 @@ class ShellFwdClient: intercept_shell: `bool` Controls whether the shell executing the script should be - executed with LD_PRELOAD=libgkfs_intercept.so (default: True). + executed with LD_PRELOAD=libgkfwd_intercept.so (default: True). timeout: `int` How much time, in seconds, we should give the process to complete. -- GitLab From caab982029ab479f52eef4a42d5c6a0ef0a4bad8 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 25 May 2020 23:36:38 -0300 Subject: [PATCH 51/71] Include missing file for the forwarding tests --- tests/integration/harness/factory.py | 58 ++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 tests/integration/harness/factory.py diff --git a/tests/integration/harness/factory.py b/tests/integration/harness/factory.py new file mode 100644 index 000000000..ef2e0585f --- /dev/null +++ b/tests/integration/harness/factory.py @@ -0,0 +1,58 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# SPDX-License-Identifier: MIT # +################################################################################ + +import os, re, hashlib +from harness.logger import logger +from harness.gkfs import FwdDaemon, FwdClient + +class FwdDaemonCreator: + """ + Factory that allows tests to create forwarding daemons in a workspace. + """ + + def __init__(self, interface, workspace): + self._interface = interface + self._workspace = workspace + + def create(self): + """ + Create a forwarding daemon in the tests workspace. + + Returns + ------- + The `FwdDaemon` object to interact with the daemon. + """ + + daemon = FwdDaemon(self._interface, self._workspace) + daemon.run() + + return daemon + +class FwdClientCreator: + """ + Factory that allows tests to create forwarding daemons in a workspace. + """ + + def __init__(self, workspace): + self._workspace = workspace + + def create(self, identifier): + """ + Create a forwarding client in the tests workspace. + + Returns + ------- + The `FwdClient` object to interact with the daemon. + """ + + return FwdClient(self._workspace, identifier) -- GitLab From eeb9b03d2456789eac1982e22ec06a7a469c3a2d Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 25 May 2020 23:58:22 -0300 Subject: [PATCH 52/71] Include missing file for the forwarding tests --- tests/integration/pytest.ini.in | 2 +- tests/integration/pytest.install.ini.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/pytest.ini.in b/tests/integration/pytest.ini.in index 3edd1784c..d7901fa8f 100644 --- a/tests/integration/pytest.ini.in +++ b/tests/integration/pytest.ini.in @@ -13,5 +13,5 @@ [pytest] testpaths = @CMAKE_CURRENT_SOURCE_DIR@ -addopts = @PYTEST_BINDIR_ARGS@ @PYTEST_LIBDIR_ARGS@ --interface=@GKFS_TESTS_INTERFACE@ --forwarding=@GKFS_TESTS_FORWARDING@ +addopts = @PYTEST_BINDIR_ARGS@ @PYTEST_LIBDIR_ARGS@ --interface=@GKFS_TESTS_INTERFACE@ enable_assertion_pass_hook = true diff --git a/tests/integration/pytest.install.ini.in b/tests/integration/pytest.install.ini.in index a8668942b..23e87ec25 100644 --- a/tests/integration/pytest.install.ini.in +++ b/tests/integration/pytest.install.ini.in @@ -13,5 +13,5 @@ [pytest] testpaths = @CMAKE_INSTALL_FULL_DATAROOTDIR@/gkfs/tests -addopts = --bin-dir=@CMAKE_INSTALL_FULL_BINDIR@ --bin-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --lib-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --interface=@GKFS_TESTS_INTERFACE@ --forwarding=@ENABLE_FORWARDING@ +addopts = --bin-dir=@CMAKE_INSTALL_FULL_BINDIR@ --bin-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --lib-dir=@CMAKE_INSTALL_FULL_LIBDIR@ --interface=@GKFS_TESTS_INTERFACE@ enable_assertion_pass_hook = true -- GitLab From 21a06e941ad712a3c40c5770ff3c7dfaf9648d4d Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 26 May 2020 00:22:40 -0300 Subject: [PATCH 53/71] Update CI configuration as we have two targets --- .gitlab-ci.yml | 189 ++++++------------------------------------------- 1 file changed, 20 insertions(+), 169 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5efd5759f..efb6050b0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,38 +3,25 @@ stages: - build deps - build - test - - test forwarding variables: - DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" - DEPS_INSTALL_PATH: "${CI_PROJECT_DIR}/deps/install" - DEPS_COMMIT: "${CI_PROJECT_DIR}/deps/install/gkfs_deps_commit" - - BUILD_PATH: "${CI_PROJECT_DIR}/gkfs/build" - INSTALL_PATH: "${CI_PROJECT_DIR}/gkfs/install" - INTEGRATION_TESTS_BIN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration" - INTEGRATION_TESTS_RUN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/run" - - BUILD_FWD_PATH: "${CI_PROJECT_DIR}/gkfwd/build" - INSTALL_FWD_PATH: "${CI_PROJECT_DIR}/gkfwd/install" - INTEGRATION_TESTS_FWD_BIN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration" - INTEGRATION_TESTS_FWD_RUN_PATH: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/run" - - TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build-gkfs" - PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" - - TESTS_BUILD_FWD_PATH: "${CI_PROJECT_DIR}/test/build-gkfwd" - PYTEST_FWD: "${CI_PROJECT_DIR}/gkfwd/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" - - LOG_PATH: "${CI_PROJECT_DIR}/logs" - - LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/deps/install/lib;${CI_PROJECT_DIR}/deps/install/lib64" + DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" + DEPS_INSTALL_PATH: "${CI_PROJECT_DIR}/deps/install" + DEPS_COMMIT: "${CI_PROJECT_DIR}/deps/install/gkfs_deps_commit" + BUILD_PATH: "${CI_PROJECT_DIR}/gkfs/build" + INSTALL_PATH: "${CI_PROJECT_DIR}/gkfs/install" + INTEGRATION_TESTS_BIN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration" + INTEGRATION_TESTS_RUN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/run" + TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build" + PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" + LOG_PATH: "${CI_PROJECT_DIR}/logs" + LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/deps/install/lib;${CI_PROJECT_DIR}/deps/install/lib64" # Configuration variables - GKFS_LOG_LEVEL: "100" - GKFS_DAEMON_LOG_PATH: "${CI_PROJECT_DIR}/logs/daemon.log" - LIBGKFS_LOG: "all" - LIBGKFS_LOG_OUTPUT: "${CI_PROJECT_DIR}/logs/gkfs_client.log" - GIT_SUBMODULE_STRATEGY: recursive + GKFS_LOG_LEVEL: "100" + GKFS_DAEMON_LOG_PATH: "${CI_PROJECT_DIR}/logs/daemon.log" + LIBGKFS_LOG: "all" + LIBGKFS_LOG_OUTPUT: "${CI_PROJECT_DIR}/logs/gkfs_client.log" + GIT_SUBMODULE_STRATEGY: recursive # Temporary new image file # image: gekkofs/gekkofs:build_env-0.8.0 @@ -43,7 +30,7 @@ image: jeanbez/gekkofs-forwarding:latest compile dependencies: stage: build deps cache: - key: deps-build + key: deps-cache paths: - ${DEPS_INSTALL_PATH}/ script: @@ -71,6 +58,8 @@ compile GekkoFS: -DCMAKE_BUILD_TYPE=Debug -DGKFS_BUILD_TESTS:BOOL=ON -DGKFS_INSTALL_TESTS:BOOL=ON + -DGFKS_ENABLE_FORWARDING:BOOL=ON + -DGFKS_ENABLE_AGIOS:BOOL=ON -DRPC_PROTOCOL="ofi+sockets" -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} @@ -80,30 +69,7 @@ compile GekkoFS: paths: - ${INSTALL_PATH} -compile GekkoFWD: - stage: build - dependencies: - - "compile dependencies" - script: - - mkdir -p ${BUILD_FWD_PATH} && cd ${BUILD_FWD_PATH} - - cmake - -Wdev - -Wdeprecate - -DCMAKE_BUILD_TYPE=Debug - -DGKFS_BUILD_TESTS:BOOL=ON - -DGKFS_INSTALL_TESTS:BOOL=ON - -DRPC_PROTOCOL="ofi+sockets" - -DGFKS_ENABLE_FORWARDING:BOOL=ON - -DGFKS_ENABLE_AGIOS:BOOL=ON - -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} - -DCMAKE_INSTALL_PREFIX=${INSTALL_FWD_PATH} - ${CI_PROJECT_DIR} - - make -j$(nproc) install - artifacts: - paths: - - ${INSTALL_FWD_PATH} - -tests GekkoFS: +compile tests: stage: build dependencies: - "compile dependencies" @@ -115,18 +81,6 @@ tests GekkoFS: paths: - ${TESTS_BUILD_PATH} -tests GekkoFWD: - stage: build - dependencies: - - "compile dependencies" - script: - - mkdir -p ${TESTS_BUILD_FWD_PATH} && cd ${TESTS_BUILD_FWD_PATH} - - cmake -DCMAKE_BUILD_TYPE=Debug .. - - make -j$(nproc) - artifacts: - paths: - - ${TESTS_BUILD_FWD_PATH} - integration tests: stage: test script: @@ -138,17 +92,6 @@ integration tests: paths: - "${INTEGRATION_TESTS_RUN_PATH}" -integration FWD tests: - stage: test forwarding - script: - - mkdir -p ${INTEGRATION_TESTS_FWD_RUN_PATH} - - cd ${INTEGRATION_TESTS_FWD_BIN_PATH} - - TMPDIR=${INTEGRATION_TESTS_FWD_RUN_PATH} unbuffer ${PYTEST} -v | tee ${INTEGRATION_TESTS_FWD_RUN_PATH}/session.log - artifacts: - when: on_failure - paths: - - "${INTEGRATION_TESTS_FWD_RUN_PATH}" - test wr: stage: test script: @@ -208,95 +151,3 @@ test lseek: when: on_failure paths: - "${LOG_PATH}" - -test FWD wr: - stage: test forwarding - dependencies: - - "compile dependencies" - - "compile GekkoFWD" - - "tests GekkoFWD" - script: - - mkdir -p "${LOG_PATH}" - - cp config/GekkoFWD/agios/* /tmp - - echo "`hostname` 0" > gkfs_forwarding.map - - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - - sleep 4 - - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_wr - artifacts: - when: on_failure - paths: - - "${LOG_PATH}" - -test FWD directories: - stage: test forwarding - dependencies: - - "compile dependencies" - - "compile GekkoFWD" - - "tests GekkoFWD" - script: - - mkdir -p "${LOG_PATH}" - - cp config/GekkoFWD/agios/* /tmp - - echo "`hostname` 0" > gkfs_forwarding.map - - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - - sleep 4 - - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_dir - artifacts: - when: on_failure - paths: - - "${LOG_PATH}" - -test FWD truncate: - stage: test forwarding - dependencies: - - "compile dependencies" - - "compile GekkoFWD" - - "tests GekkoFWD" - script: - - mkdir -p "${LOG_PATH}" - - cp config/GekkoFWD/agios/* /tmp - - echo "`hostname` 0" > gkfs_forwarding.map - - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - - sleep 4 - - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_truncate - artifacts: - when: on_failure - paths: - - "${LOG_PATH}" - -test FWD path resolution: - stage: test forwarding - dependencies: - - "compile dependencies" - - "compile GekkoFWD" - - "tests GekkoFWD" - script: - - mkdir -p "${LOG_PATH}" - - cp config/GekkoFWD/agios/* /tmp - - echo "`hostname` 0" > gkfs_forwarding.map - - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - - sleep 4 - - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_path_resolution - artifacts: - when: on_failure - paths: - - "${LOG_PATH}" - -test FWD lseek: - stage: test forwarding - dependencies: - - "compile dependencies" - - "compile GekkoFWD" - - "tests GekkoFWD" - script: - - mkdir -p "${LOG_PATH}" - - cp config/GekkoFWD/agios/* /tmp - - echo "`hostname` 0" > gkfs_forwarding.map - - ${INSTALL_FWD_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - - sleep 4 - - LD_PRELOAD=${INSTALL_FWD_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_FWD_PATH}/gkfs_test_lseek - artifacts: - when: on_failure - paths: - - "${LOG_PATH}" - - -- GitLab From 44c14fb2924b09b51ca86aa8540d2f671f112015 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 26 May 2020 01:07:27 -0300 Subject: [PATCH 54/71] Fix typo in environment variable --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index efb6050b0..9e69feb0c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,8 +58,8 @@ compile GekkoFS: -DCMAKE_BUILD_TYPE=Debug -DGKFS_BUILD_TESTS:BOOL=ON -DGKFS_INSTALL_TESTS:BOOL=ON - -DGFKS_ENABLE_FORWARDING:BOOL=ON - -DGFKS_ENABLE_AGIOS:BOOL=ON + -DGKFS_ENABLE_FORWARDING:BOOL=ON + -DGKFS_ENABLE_AGIOS:BOOL=ON -DRPC_PROTOCOL="ofi+sockets" -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} -- GitLab From 87423f89457a475b04182691297ef552e80deb0d Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 27 May 2020 00:54:28 -0300 Subject: [PATCH 55/71] Removed unused option --- tests/integration/conftest.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 232bf209c..5d7ba9c4e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -100,8 +100,6 @@ def gkfs_client(test_workspace, request): be requested from a co-running daemon. """ - interface = request.config.getoption('--interface') - return Client(test_workspace) @pytest.fixture @@ -111,8 +109,6 @@ def gkfs_shell(test_workspace, request): (stat, ls, mkdir, etc.) can be issued to a co-running daemon. """ - interface = request.config.getoption('--interface') - return ShellClient(test_workspace) @pytest.fixture -- GitLab From 9184410651c3ba6507fb084b1bc694395c176190 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Fri, 29 May 2020 10:37:19 -0300 Subject: [PATCH 56/71] Include new forwarding tests --- include/global/rpc/distributor.hpp | 1 + src/global/rpc/distributor.cpp | 5 +- tests/integration/forwarding/test_map.py | 90 +++++++++++++++++++++++- tests/integration/harness/gkfs.py | 18 ++++- 4 files changed, 109 insertions(+), 5 deletions(-) diff --git a/include/global/rpc/distributor.hpp b/include/global/rpc/distributor.hpp index 86b85e6fa..d743e580d 100644 --- a/include/global/rpc/distributor.hpp +++ b/include/global/rpc/distributor.hpp @@ -72,6 +72,7 @@ public: class ForwarderDistributor : public Distributor { private: host_t fwd_host_; + std::vector all_hosts_; unsigned int hosts_size_; std::hash str_hash; public: diff --git a/src/global/rpc/distributor.cpp b/src/global/rpc/distributor.cpp index cbb2f428f..449103787 100644 --- a/src/global/rpc/distributor.cpp +++ b/src/global/rpc/distributor.cpp @@ -71,7 +71,8 @@ locate_directory_metadata(const string& path) const { ForwarderDistributor:: ForwarderDistributor(host_t fwhost, unsigned int hosts_size) : fwd_host_(fwhost), - hosts_size_(hosts_size) + hosts_size_(hosts_size), + all_hosts_(hosts_size) {} host_t ForwarderDistributor:: @@ -91,7 +92,7 @@ locate_file_metadata(const std::string& path) const { std::vector ForwarderDistributor:: locate_directory_metadata(const std::string& path) const { - return {fwd_host_}; + return all_hosts_; } } // namespace rpc } // namespace gkfs diff --git a/tests/integration/forwarding/test_map.py b/tests/integration/forwarding/test_map.py index 41d0507a1..855d1c177 100644 --- a/tests/integration/forwarding/test_map.py +++ b/tests/integration/forwarding/test_map.py @@ -18,6 +18,7 @@ import stat import os import time import ctypes +import socket import sh import sys import pytest @@ -26,7 +27,7 @@ from harness.logger import logger nonexisting = "nonexisting" -def test_write_two_io_nodes(gkfwd_daemon_factory, gkfwd_client_factory): +def test_two_io_nodes(gkfwd_daemon_factory, gkfwd_client_factory): """Write files from two clients using two daemons""" d00 = gkfwd_daemon_factory.create() @@ -111,4 +112,89 @@ def test_write_two_io_nodes(gkfwd_daemon_factory, gkfwd_client_factory): assert ret.dirents[1].d_name == 'file-c01' assert ret.dirents[1].d_type == 8 # DT_REG - assert ret.errno == 115 #FIXME: Should be 0! \ No newline at end of file + assert ret.errno == 115 #FIXME: Should be 0! + + with open(c00.log) as f: + lines = f.readlines() + + for line in lines: + if 'Forward to' in line: + ion = line.split()[-1] + + assert ion == '0' + + with open(c01.log) as f: + lines = f.readlines() + + for line in lines: + if 'Forward to' in line: + ion = line.split()[-1] + + assert ion == '1' + +def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): + """Write files from two clients using two daemons""" + + d00 = gkfwd_daemon_factory.create() + d01 = gkfwd_daemon_factory.create() + + c00 = gkfwd_client_factory.create('c-0') + c01 = gkfwd_client_factory.create('c-1') + + file = d00.mountdir / "file-c00-1" + + # create a file in gekkofs + ret = c00.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf = b'42' + ret = c00.write(file, buf, len(buf)) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + with open(c00.log) as f: + lines = f.readlines() + + for line in lines: + if 'Forward to' in line: + ion = line.split()[-1] + + assert ion == '0' + + # recreate the mapping so that the server that wrote will now read + c00.remap('c-1') + + # we need to wait for at least the number of seconds between remap calls + time.sleep(10) + + file = d00.mountdir / "file-c00-2" + + # open the file to write + ret = c00.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + buf = b'24' + ret = c00.write(file, buf, len(buf)) + + assert ret.retval == len(buf) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! + + with open(c00.log) as f: + lines = f.readlines() + + for line in lines: + if 'Forward to' in line: + ion = line.split()[-1] + + assert ion == '1' diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index 72308298d..ca4d22dd3 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -751,6 +751,13 @@ class FwdClient: fwd_map_file.write('{} {}\n'.format(socket.gethostname(), int(identifier.split('-')[1]))) fwd_map_file.close() + # record the map so we can modify it latter if needed + self._map = self.cwd / gkfwd_forwarding_map_file_local + + # we need to ensure each client will have a distinct log + gkfwd_client_log_file_local = '{}-{}'.format(identifier, gkfwd_client_log_file) + self._log = self._workspace.logdir / gkfwd_client_log_file_local + libdirs = ':'.join( filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + [str(p) for p in self._workspace.libdirs])) @@ -783,7 +790,7 @@ class FwdClient: 'LIBGKFS_HOSTS_FILE' : self.cwd / gkfwd_hosts_file, 'LIBGKFS_FORWARDING_MAP_FILE' : self.cwd / gkfwd_forwarding_map_file_local, 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : self._workspace.logdir / gkfwd_client_log_file + 'LIBGKFS_LOG_OUTPUT' : self._workspace.logdir / gkfwd_client_log_file_local } self._env.update(self._patched_env) @@ -812,6 +819,11 @@ class FwdClient: logger.debug(f"command output: {out.stdout}") return self._parser.parse(cmd, out.stdout) + def remap(self, identifier): + fwd_map_file = open(self.cwd / self._map, 'w') + fwd_map_file.write('{} {}\n'.format(socket.gethostname(), int(identifier.split('-')[1]))) + fwd_map_file.close() + def __getattr__(self, name): return _proxy_exec(self, name) @@ -819,6 +831,10 @@ class FwdClient: def cwd(self): return self._workspace.twd + @property + def log(self): + return self._log + class ShellFwdClient: """ A class to represent a GekkoFS shell client process. -- GitLab From 29fbfd151f9fec1191bd016ece890eae4af6ff25 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 9 Jun 2020 13:54:13 -0300 Subject: [PATCH 57/71] Updates on forwarding root location and mapping thread shutdown --- include/config.hpp | 4 ++++ include/global/rpc/distributor.hpp | 2 +- src/client/preload.cpp | 14 +++++++++++++- src/daemon/daemon.cpp | 5 +++++ src/global/rpc/distributor.cpp | 5 +++-- 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/include/config.hpp b/include/config.hpp index 83b5f4bc2..6ed084fec 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -52,7 +52,11 @@ constexpr auto use_blocks = false; } // namespace metadata namespace rpc { +#ifdef GKFS_ENABLE_FORWARDING +constexpr auto chunksize = 107374182400; // in bytes (e.g., 107374182400 == 100GB) +#else constexpr auto chunksize = 524288; // in bytes (e.g., 524288 == 512KB) +#endif //size of preallocated buffer to hold directory entries in rpc call constexpr auto dirents_buff_size = (8 * 1024 * 1024); // 8 mega /* diff --git a/include/global/rpc/distributor.hpp b/include/global/rpc/distributor.hpp index d743e580d..d01a79412 100644 --- a/include/global/rpc/distributor.hpp +++ b/include/global/rpc/distributor.hpp @@ -72,8 +72,8 @@ public: class ForwarderDistributor : public Distributor { private: host_t fwd_host_; - std::vector all_hosts_; unsigned int hosts_size_; + std::vector all_hosts_; std::hash str_hash; public: ForwarderDistributor(host_t fwhost, unsigned int hosts_size); diff --git a/src/client/preload.cpp b/src/client/preload.cpp index e687bf488..cad952c9b 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -39,6 +39,9 @@ pthread_once_t init_env_thread = PTHREAD_ONCE_INIT; #ifdef GKFS_ENABLE_FORWARDING pthread_t mapper; bool forwarding_running; + +pthread_mutex_t remap_mutex; +pthread_cond_t remap_signal; #endif inline void exit_error_msg(int errcode, const string& msg) { @@ -130,6 +133,10 @@ void init_ld_environment_() { #ifdef GKFS_ENABLE_FORWARDING void *forwarding_mapper(void* p) { + struct timespec timeout; + clock_gettime(CLOCK_REALTIME, &timeout); + timeout.tv_sec += 10; // 10 seconds + while (forwarding_running) { try { gkfs::util::load_forwarding_map(); @@ -140,7 +147,10 @@ void *forwarding_mapper(void* p) { } // Sleeps for 10 seconds - sleep(10); + // sleep(10); + pthread_mutex_lock(&remap_mutex); + pthread_cond_timedwait(&remap_signal, &remap_mutex, &timeout); + pthread_mutex_unlock(&remap_mutex); } return nullptr; @@ -159,6 +169,8 @@ void init_forwarding_mapper() { void destroy_forwarding_mapper() { forwarding_running = false; + pthread_cond_signal(&remap_signal); + pthread_join(mapper, NULL); } #endif diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 59e9df6ef..4758fd2df 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -403,7 +403,12 @@ int main(int argc, const char* argv[]) { assert(vm.count("rootdir")); auto rootdir = vm["rootdir"].as(); + #ifdef GKFS_ENABLE_FORWARDING + // In forwarding mode, the backend is shared + auto rootdir_path = bfs::path(rootdir); + #else auto rootdir_path = bfs::path(rootdir) / fmt::format_int(getpid()).str(); + #endif GKFS_DATA->spdlogger()->debug("{}() Root directory: '{}'", __func__, rootdir_path.native()); bfs::create_directories(rootdir_path); diff --git a/src/global/rpc/distributor.cpp b/src/global/rpc/distributor.cpp index 449103787..ba8d700cf 100644 --- a/src/global/rpc/distributor.cpp +++ b/src/global/rpc/distributor.cpp @@ -72,8 +72,9 @@ ForwarderDistributor:: ForwarderDistributor(host_t fwhost, unsigned int hosts_size) : fwd_host_(fwhost), hosts_size_(hosts_size), - all_hosts_(hosts_size) -{} + all_hosts_(hosts_size) { + ::iota(all_hosts_.begin(), all_hosts_.end(), 0); +} host_t ForwarderDistributor:: localhost() const { -- GitLab From c6dd5ce7c2d29d4564d11a1686a0bb737c56a5a1 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 9 Jun 2020 20:06:40 -0300 Subject: [PATCH 58/71] Include new forwarding test --- include/config.hpp | 4 -- src/daemon/daemon.cpp | 15 ++++ tests/integration/forwarding/test_map.py | 92 ++++++++++++++++++++++++ tests/integration/harness/gkfs.py | 5 ++ tests/integration/harness/workspace.py | 5 ++ 5 files changed, 117 insertions(+), 4 deletions(-) diff --git a/include/config.hpp b/include/config.hpp index 6ed084fec..83b5f4bc2 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -52,11 +52,7 @@ constexpr auto use_blocks = false; } // namespace metadata namespace rpc { -#ifdef GKFS_ENABLE_FORWARDING -constexpr auto chunksize = 107374182400; // in bytes (e.g., 107374182400 == 100GB) -#else constexpr auto chunksize = 524288; // in bytes (e.g., 524288 == 512KB) -#endif //size of preallocated buffer to hold directory entries in rpc call constexpr auto dirents_buff_size = (8 * 1024 * 1024); // 8 mega /* diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 4758fd2df..8a8589184 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -416,11 +416,26 @@ int main(int argc, const char* argv[]) { if (vm.count("metadir")) { auto metadir = vm["metadir"].as(); + + #ifdef GKFS_ENABLE_FORWARDING + auto metadir_path = bfs::path(metadir) / fmt::format_int(getpid()).str(); + bfs::create_directories(metadir_path); + GKFS_DATA->metadir(bfs::canonical(metadir_path).native()); + #else bfs::create_directories(metadir); GKFS_DATA->metadir(bfs::canonical(metadir).native()); + #endif } else { // use rootdir as metadata dir + auto metadir = vm["rootdir"].as(); + + #ifdef GKFS_ENABLE_FORWARDING + auto metadir_path = bfs::path(metadir) / fmt::format_int(getpid()).str(); + bfs::create_directories(metadir_path); + GKFS_DATA->metadir(bfs::canonical(metadir_path).native()); + #else GKFS_DATA->metadir(GKFS_DATA->rootdir()); + #endif } try { diff --git a/tests/integration/forwarding/test_map.py b/tests/integration/forwarding/test_map.py index 855d1c177..5a2d33a0b 100644 --- a/tests/integration/forwarding/test_map.py +++ b/tests/integration/forwarding/test_map.py @@ -198,3 +198,95 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): ion = line.split()[-1] assert ion == '1' + +def test_two_io_nodes_operations(gkfwd_daemon_factory, gkfwd_client_factory): + """Write files from one client and read in the other using two daemons""" + + d00 = gkfwd_daemon_factory.create() + d01 = gkfwd_daemon_factory.create() + + c00 = gkfwd_client_factory.create('c-0') + c01 = gkfwd_client_factory.create('c-1') + + file = d00.mountdir / "file-c00" + + # create a file in gekkofs + ret = c00.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf = b'42' + ret = c00.write(file, buf, len(buf)) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = c00.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + ret = c00.read(file, len(buf)) + + assert ret.buf == buf + assert ret.retval == len(buf) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = c01.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + ret = c01.read(file, len(buf)) + + assert ret.buf == buf + assert ret.retval == len(buf) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # the file should be there and accessible by the two clients + ret = c00.readdir(d00.mountdir) + + assert len(ret.dirents) == 1 + + assert ret.dirents[0].d_name == 'file-c00' + assert ret.dirents[0].d_type == 8 # DT_REG + assert ret.errno == 115 #FIXME: Should be 0! + + # the file should be there and accessible by the two clients + ret = c01.readdir(d01.mountdir) + + assert len(ret.dirents) == 1 + + assert ret.dirents[0].d_name == 'file-c00' + assert ret.dirents[0].d_type == 8 # DT_REG + assert ret.errno == 115 #FIXME: Should be 0! + + with open(c00.log) as f: + lines = f.readlines() + + for line in lines: + if 'Forward to' in line: + ion = line.split()[-1] + + assert ion == '0' + + with open(c01.log) as f: + lines = f.readlines() + + for line in lines: + if 'Forward to' in line: + ion = line.split()[-1] + + assert ion == '1' \ No newline at end of file diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index ca4d22dd3..e2e0610c5 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -619,6 +619,7 @@ class FwdDaemon: def run(self): args = [ '--mountdir', self.mountdir, + '--metadir', self.metadir, '--rootdir', self.rootdir, '-l', self._address ] @@ -717,6 +718,10 @@ class FwdDaemon: def rootdir(self): return self._workspace.rootdir + @property + def metadir(self): + return self._workspace.metadir + @property def mountdir(self): return self._workspace.mountdir diff --git a/tests/integration/harness/workspace.py b/tests/integration/harness/workspace.py index 319b55a90..50f135c74 100644 --- a/tests/integration/harness/workspace.py +++ b/tests/integration/harness/workspace.py @@ -48,6 +48,7 @@ class Workspace: self._libdirs = libdirs self._logdir = self._twd / 'logs' self._rootdir = self._twd / 'root' + self._metadir = self._twd / 'meta' self._mountdir = self._twd / 'mnt' self._tmpdir = self._twd / 'tmp' @@ -76,6 +77,10 @@ class Workspace: def rootdir(self): return self._rootdir + @property + def metadir(self): + return self._metadir + @property def mountdir(self): return self._mountdir -- GitLab From b771b6d4021ace2497ad6ee5a66717aaf1d18edb Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 9 Jun 2020 21:38:50 -0300 Subject: [PATCH 59/71] Include new forwarding test --- src/client/preload.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/client/preload.cpp b/src/client/preload.cpp index cad952c9b..231a4a6b5 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -146,8 +146,6 @@ void *forwarding_mapper(void* p) { exit_error_msg(EXIT_FAILURE, fmt::format("Unable set the forwarding host '{}'", e.what())); } - // Sleeps for 10 seconds - // sleep(10); pthread_mutex_lock(&remap_mutex); pthread_cond_timedwait(&remap_signal, &remap_mutex, &timeout); pthread_mutex_unlock(&remap_mutex); -- GitLab From 7aca27f5e5a0417947641b44ee135378b32c8c7d Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 10 Jun 2020 00:26:49 -0300 Subject: [PATCH 60/71] Include new forwarding test --- src/daemon/daemon.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 8a8589184..24f15c603 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -419,12 +419,15 @@ int main(int argc, const char* argv[]) { #ifdef GKFS_ENABLE_FORWARDING auto metadir_path = bfs::path(metadir) / fmt::format_int(getpid()).str(); - bfs::create_directories(metadir_path); - GKFS_DATA->metadir(bfs::canonical(metadir_path).native()); #else - bfs::create_directories(metadir); - GKFS_DATA->metadir(bfs::canonical(metadir).native()); + auto metadir_path = bfs::path(metadir); #endif + + bfs::create_directories(metadir_path); + GKFS_DATA->metadir(bfs::canonical(metadir_path).native()); + + GKFS_DATA->spdlogger()->debug("{}() Meta directory: '{}'", + __func__, metadir_path.native()); } else { // use rootdir as metadata dir auto metadir = vm["rootdir"].as(); -- GitLab From 72f5d3764bccd7ae944ee31fb3d48e0926fd1550 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 10 Jun 2020 11:54:25 -0300 Subject: [PATCH 61/71] Remove duplicate code from CMakeLists --- src/client/CMakeLists.txt | 8 +++----- src/daemon/CMakeLists.txt | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 44429c726..0d2ce4585 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -133,11 +133,9 @@ if(GKFS_ENABLE_FORWARDING) add_library(gkfwd_intercept SHARED ${FWD_PRELOAD_SRC} ${FWD_PRELOAD_HEADERS}) - if(GKFS_ENABLE_FORWARDING) - set_target_properties(gkfwd_intercept - PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING - ) - endif() + set_target_properties(gkfwd_intercept + PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING + ) message(STATUS "[gekkofs] Forwarding mode: ${GKFS_ENABLE_FORWARDING}") if(GKFS_ENABLE_AGIOS) diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index e63b0bd00..c33309609 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -95,11 +95,9 @@ if(GKFS_ENABLE_FORWARDING) ) add_executable(gkfwd_daemon ${FWD_DAEMON_SRC} ${FWD_DAEMON_HEADERS}) - if(GKFS_ENABLE_FORWARDING) - set_target_properties(gkfwd_daemon - PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING - ) - endif() + set_target_properties(gkfwd_daemon + PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING + ) message(STATUS "[gekkofs] Forwarding mode: ${GKFS_ENABLE_FORWARDING}") if(GKFS_ENABLE_AGIOS) -- GitLab From fa12257c1f123e42d4b387247d4888593fea3fc9 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 10 Jun 2020 14:00:13 -0300 Subject: [PATCH 62/71] Update CMakeLists --- src/daemon/CMakeLists.txt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index c33309609..72128f564 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -95,16 +95,17 @@ if(GKFS_ENABLE_FORWARDING) ) add_executable(gkfwd_daemon ${FWD_DAEMON_SRC} ${FWD_DAEMON_HEADERS}) - set_target_properties(gkfwd_daemon - PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING - ) - message(STATUS "[gekkofs] Forwarding mode: ${GKFS_ENABLE_FORWARDING}") - if(GKFS_ENABLE_AGIOS) set_target_properties(gkfwd_daemon - PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_AGIOS + PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING -DGKFS_ENABLE_AGIOS ) + else() + set_target_properties(gkfwd_daemon + PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING + ) endif() + + message(STATUS "[gekkofs] Forwarding mode: ${GKFS_ENABLE_FORWARDING}") message(STATUS "[gekkofs] AGIOS scheduling: ${GKFS_ENABLE_AGIOS}") target_link_libraries(gkfwd_daemon -- GitLab From 675f6572248adaecf35f832d3def3665bb13478e Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 10 Jun 2020 21:01:10 -0300 Subject: [PATCH 63/71] Fix syntax --- src/daemon/CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 72128f564..ad0829c4b 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -97,11 +97,14 @@ if(GKFS_ENABLE_FORWARDING) if(GKFS_ENABLE_AGIOS) set_target_properties(gkfwd_daemon - PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING -DGKFS_ENABLE_AGIOS + PROPERTIES + COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING + COMPILE_FLAGS -DGKFS_ENABLE_AGIOS ) else() set_target_properties(gkfwd_daemon - PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING + PROPERTIES + COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING ) endif() -- GitLab From fd2be6f82f7f950a829db6f690f9384522fa601c Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 10 Jun 2020 23:19:04 -0300 Subject: [PATCH 64/71] Fix syntax --- src/client/CMakeLists.txt | 18 +++++++++++------- src/daemon/CMakeLists.txt | 16 ++++++++-------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 0d2ce4585..d477ec99c 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -133,16 +133,20 @@ if(GKFS_ENABLE_FORWARDING) add_library(gkfwd_intercept SHARED ${FWD_PRELOAD_SRC} ${FWD_PRELOAD_HEADERS}) - set_target_properties(gkfwd_intercept - PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING - ) - message(STATUS "[gekkofs] Forwarding mode: ${GKFS_ENABLE_FORWARDING}") - if(GKFS_ENABLE_AGIOS) - set_target_properties(gkfwd_intercept - PROPERTIES COMPILE_FLAGS -DGKFS_ENABLE_AGIOS + target_compile_definitions(gkfwd_daemon + PUBLIC + GKFS_ENABLE_FORWARDING + DGKFS_ENABLE_AGIOS + ) + else() + target_compile_definitions(gkfwd_daemon + PUBLIC + GKFS_ENABLE_FORWARDING ) endif() + + message(STATUS "[gekkofs] Forwarding mode: ${GKFS_ENABLE_FORWARDING}") message(STATUS "[gekkofs] AGIOS scheduling: ${GKFS_ENABLE_AGIOS}") target_link_libraries(gkfwd_intercept diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index ad0829c4b..da9edfcf1 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -96,16 +96,16 @@ if(GKFS_ENABLE_FORWARDING) add_executable(gkfwd_daemon ${FWD_DAEMON_SRC} ${FWD_DAEMON_HEADERS}) if(GKFS_ENABLE_AGIOS) - set_target_properties(gkfwd_daemon - PROPERTIES - COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING - COMPILE_FLAGS -DGKFS_ENABLE_AGIOS + target_compile_definitions(gkfwd_daemon + PUBLIC + GKFS_ENABLE_FORWARDING + DGKFS_ENABLE_AGIOS ) else() - set_target_properties(gkfwd_daemon - PROPERTIES - COMPILE_FLAGS -DGKFS_ENABLE_FORWARDING - ) + target_compile_definitions(gkfwd_daemon + PUBLIC + GKFS_ENABLE_FORWARDING + ) endif() message(STATUS "[gekkofs] Forwarding mode: ${GKFS_ENABLE_FORWARDING}") -- GitLab From d62be5480224ea5116b513f38a3fa3ae7754f8a8 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Thu, 11 Jun 2020 13:46:56 -0300 Subject: [PATCH 65/71] Remove message at every iteration --- src/client/preload.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 231a4a6b5..8f37af8fc 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -137,11 +137,17 @@ void *forwarding_mapper(void* p) { clock_gettime(CLOCK_REALTIME, &timeout); timeout.tv_sec += 10; // 10 seconds + int previous = -1; + while (forwarding_running) { try { gkfs::util::load_forwarding_map(); - LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + if (previous != CTX->fwd_host_id()) { + LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + + previous = CTX->fwd_host_id(); + } } catch (std::exception& e) { exit_error_msg(EXIT_FAILURE, fmt::format("Unable set the forwarding host '{}'", e.what())); } -- GitLab From 31c484468b39c770cac74ee23e1b9619adeb26f2 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Fri, 7 Aug 2020 15:32:44 -0300 Subject: [PATCH 66/71] Refactor code --- tests/integration/conftest.py | 3 +- tests/integration/harness/factory.py | 58 ---------------------------- tests/integration/harness/gkfs.py | 43 +++++++++++++++++++++ 3 files changed, 44 insertions(+), 60 deletions(-) delete mode 100644 tests/integration/harness/factory.py diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 5d7ba9c4e..44113e353 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -19,8 +19,7 @@ from pathlib import Path from harness.logger import logger, initialize_logging, finalize_logging from harness.cli import add_cli_options, set_default_log_formatter from harness.workspace import Workspace, FileCreator -from harness.gkfs import Daemon, Client, ShellClient, FwdDaemon, FwdClient, ShellFwdClient -from harness.factory import FwdDaemonCreator, FwdClientCreator +from harness.gkfs import Daemon, Client, ShellClient, FwdDaemon, FwdClient, ShellFwdClient, FwdDaemonCreator, FwdClientCreator from harness.reporter import report_test_status, report_test_headline, report_assertion_pass def pytest_configure(config): diff --git a/tests/integration/harness/factory.py b/tests/integration/harness/factory.py deleted file mode 100644 index ef2e0585f..000000000 --- a/tests/integration/harness/factory.py +++ /dev/null @@ -1,58 +0,0 @@ -################################################################################ -# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # -# # -# This software was partially supported by the # -# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # -# # -# This software was partially supported by the # -# ADA-FS project under the SPPEXA project funded by the DFG. # -# # -# SPDX-License-Identifier: MIT # -################################################################################ - -import os, re, hashlib -from harness.logger import logger -from harness.gkfs import FwdDaemon, FwdClient - -class FwdDaemonCreator: - """ - Factory that allows tests to create forwarding daemons in a workspace. - """ - - def __init__(self, interface, workspace): - self._interface = interface - self._workspace = workspace - - def create(self): - """ - Create a forwarding daemon in the tests workspace. - - Returns - ------- - The `FwdDaemon` object to interact with the daemon. - """ - - daemon = FwdDaemon(self._interface, self._workspace) - daemon.run() - - return daemon - -class FwdClientCreator: - """ - Factory that allows tests to create forwarding daemons in a workspace. - """ - - def __init__(self, workspace): - self._workspace = workspace - - def create(self, identifier): - """ - Create a forwarding client in the tests workspace. - - Returns - ------- - The `FwdClient` object to interact with the daemon. - """ - - return FwdClient(self._workspace, identifier) diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index e2e0610c5..07373ddb0 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -140,6 +140,49 @@ def _process_exists(pid): return True +class FwdDaemonCreator: + """ + Factory that allows tests to create forwarding daemons in a workspace. + """ + + def __init__(self, interface, workspace): + self._interface = interface + self._workspace = workspace + + def create(self): + """ + Create a forwarding daemon in the tests workspace. + + Returns + ------- + The `FwdDaemon` object to interact with the daemon. + """ + + daemon = FwdDaemon(self._interface, self._workspace) + daemon.run() + + return daemon + +class FwdClientCreator: + """ + Factory that allows tests to create forwarding daemons in a workspace. + """ + + def __init__(self, workspace): + self._workspace = workspace + + def create(self, identifier): + """ + Create a forwarding client in the tests workspace. + + Returns + ------- + The `FwdClient` object to interact with the daemon. + """ + + return FwdClient(self._workspace, identifier) + + class Daemon: def __init__(self, interface, workspace): -- GitLab From 18b08f5143f11200ca6de3db13302cf7f6e2afe4 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 17 Aug 2020 10:31:02 -0300 Subject: [PATCH 67/71] Fix merge conflicts --- CMakeLists.txt | 11 +++++++++ docker/build_env.docker | 1 + src/client/gkfs_functions.cpp | 4 ++++ src/client/preload.cpp | 40 +++++++++++++++++++++++++++++++++ src/daemon/CMakeLists.txt | 4 ++++ src/daemon/handler/srv_data.cpp | 8 +++++++ 6 files changed, 68 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index c4c1af288..3a7349f1b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,17 @@ if(ENABLE_CLIENT_LOG) add_definitions(-DGKFS_ENABLE_LOGGING) endif() message(STATUS "[gekkofs] Client logging output: ${ENABLE_CLIENT_LOG}") +option(ENABLE_FORWARDING "Enable forwarding mode" OFF) +if(ENABLE_FORWARDING) + add_definitions(-DGKFS_ENABLE_FORWARDING) +endif() +message(STATUS "[gekkofs] Forwarding mode: ${ENABLE_FORWARDING}") + +option(ENABLE_AGIOS "Enable AGIOS scheduling library" OFF) +if(ENABLE_AGIOS) + add_definitions(-DGKFS_ENABLE_AGIOS) +endif() +message(STATUS "[gekkofs] AGIOS scheduling: ${ENABLE_AGIOS}") option(GKFS_ENABLE_FORWARDING "Enable forwarding mode" OFF) option(GKFS_ENABLE_AGIOS "Enable AGIOS scheduling library" OFF) diff --git a/docker/build_env.docker b/docker/build_env.docker index 346807b86..ef83cbaa9 100644 --- a/docker/build_env.docker +++ b/docker/build_env.docker @@ -80,6 +80,7 @@ RUN yum -y -q update && yum -y -q install \ automake \ cmake \ cmake3 \ + sudo \ gcc \ gcc-c++ \ openssl-devel \ diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index a4c5507bc..39ddf5cfe 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -531,7 +531,11 @@ ssize_t gkfs_read(int fd, void* buf, size_t count) { ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) { auto file = CTX->file_map()->get(fd); +<<<<<<< Updated upstream auto pos = offset; // keep track of current position +======= + auto pos = offset; // keep truck of current position +>>>>>>> Stashed changes ssize_t read = 0; ssize_t ret; for (int i = 0; i < iovcnt; ++i) { diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 8f37af8fc..e29a78e87 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -36,6 +36,7 @@ namespace { // make sure that things are only initialized once pthread_once_t init_env_thread = PTHREAD_ONCE_INIT; +<<<<<<< Updated upstream #ifdef GKFS_ENABLE_FORWARDING pthread_t mapper; bool forwarding_running; @@ -43,6 +44,10 @@ bool forwarding_running; pthread_mutex_t remap_mutex; pthread_cond_t remap_signal; #endif +======= +pthread_t mapper; +bool forwarding_running; +>>>>>>> Stashed changes inline void exit_error_msg(int errcode, const string& msg) { @@ -132,6 +137,7 @@ void init_ld_environment_() { } #ifdef GKFS_ENABLE_FORWARDING +<<<<<<< Updated upstream void *forwarding_mapper(void* p) { struct timespec timeout; clock_gettime(CLOCK_REALTIME, &timeout); @@ -139,10 +145,14 @@ void *forwarding_mapper(void* p) { int previous = -1; +======= +void *forwarding_mapper(void *p) { +>>>>>>> Stashed changes while (forwarding_running) { try { gkfs::util::load_forwarding_map(); +<<<<<<< Updated upstream if (previous != CTX->fwd_host_id()) { LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); @@ -162,11 +172,25 @@ void *forwarding_mapper(void* p) { #endif #ifdef GKFS_ENABLE_FORWARDING +======= + LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + } catch (std::exception& e){ + exit_error_msg(EXIT_FAILURE, fmt::format("Unable set the forwarding host '{}'", e.what())); + } + + // Sleeps for 10 seconds + sleep(10); + } +} +#endif + +>>>>>>> Stashed changes void init_forwarding_mapper() { forwarding_running = true; pthread_create(&mapper, NULL, forwarding_mapper, NULL); } +<<<<<<< Updated upstream #endif #ifdef GKFS_ENABLE_FORWARDING @@ -178,6 +202,14 @@ void destroy_forwarding_mapper() { pthread_join(mapper, NULL); } #endif +======= + +void destroy_forwarding_mapper() { + forwarding_running = false; + + pthread_join(mapper, NULL); +} +>>>>>>> Stashed changes void log_prog_name() { std::string line; @@ -238,9 +270,13 @@ void init_preload() { CTX->unprotect_user_fds(); +<<<<<<< Updated upstream #ifdef GKFS_ENABLE_FORWARDING init_forwarding_mapper(); #endif +======= + init_forwarding_mapper(); +>>>>>>> Stashed changes gkfs::preload::start_interception(); } @@ -249,9 +285,13 @@ void init_preload() { * Called last when preload library is used with the LD_PRELOAD environment variable */ void destroy_preload() { +<<<<<<< Updated upstream #ifdef GKFS_ENABLE_FORWARDING destroy_forwarding_mapper(); #endif +======= + destroy_forwarding_mapper(); +>>>>>>> Stashed changes CTX->clear_hosts(); LOG(DEBUG, "Peer information deleted"); diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index da9edfcf1..8e14d17af 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -12,6 +12,7 @@ set(DAEMON_SRC handler/srv_metadata.cpp handler/srv_data.cpp handler/srv_management.cpp + scheduler/agios.cpp ) set(DAEMON_HEADERS ../../include/config.hpp @@ -23,6 +24,7 @@ set(DAEMON_HEADERS ../../include/global/path_util.hpp ../../include/daemon/daemon.hpp ../../include/daemon/util.hpp + ../../include/daemon/scheduler/agios.hpp ../../include/daemon/ops/metadentry.hpp ../../include/daemon/classes/fs_data.hpp ../../include/daemon/classes/rpc_data.hpp @@ -40,6 +42,7 @@ target_link_libraries(gkfs_daemon env_util spdlog fmt::fmt + ${AGIOS_LIBRARIES} # margo libs ${ABT_LIBRARIES} mercury @@ -53,6 +56,7 @@ target_link_libraries(gkfs_daemon target_include_directories(gkfs_daemon PRIVATE + ${AGIOS_INCLUDE_DIRS} ${ABT_INCLUDE_DIRS} ${MARGO_INCLUDE_DIRS} ) diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index 4eae87c9b..c223436cc 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -644,11 +644,19 @@ static hg_return_t rpc_srv_get_chunk_stat(hg_handle_t handle) { DEFINE_MARGO_RPC_HANDLER(rpc_srv_get_chunk_stat) #ifdef GKFS_ENABLE_AGIOS +<<<<<<< Updated upstream void *agios_eventual_callback(int64_t request_id, void* info) { GKFS_DATA->spdlogger()->debug("{}() custom callback request {} is ready", __func__, request_id); ABT_eventual_set((ABT_eventual) info, &request_id, sizeof(int64_t)); +======= +void *agios_eventual_callback(int64_t request_id, void *info) { + GKFS_DATA->spdlogger()->debug("{}() custom callback request {} is ready", __func__, request_id); + + ABT_eventual_set((ABT_eventual) info, &request_id, sizeof(int64_t)); + +>>>>>>> Stashed changes return 0; } #endif -- GitLab From 8020da4a9824f8ac94c451ac9fe767e18efb6205 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 17 Aug 2020 10:58:24 -0300 Subject: [PATCH 68/71] Fix merge conflicts --- CMakeLists.txt | 11 --------- src/client/gkfs_functions.cpp | 4 ---- src/client/preload.cpp | 40 --------------------------------- src/daemon/CMakeLists.txt | 6 +---- src/daemon/handler/srv_data.cpp | 8 ------- 5 files changed, 1 insertion(+), 68 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a7349f1b..c4c1af288 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,17 +142,6 @@ if(ENABLE_CLIENT_LOG) add_definitions(-DGKFS_ENABLE_LOGGING) endif() message(STATUS "[gekkofs] Client logging output: ${ENABLE_CLIENT_LOG}") -option(ENABLE_FORWARDING "Enable forwarding mode" OFF) -if(ENABLE_FORWARDING) - add_definitions(-DGKFS_ENABLE_FORWARDING) -endif() -message(STATUS "[gekkofs] Forwarding mode: ${ENABLE_FORWARDING}") - -option(ENABLE_AGIOS "Enable AGIOS scheduling library" OFF) -if(ENABLE_AGIOS) - add_definitions(-DGKFS_ENABLE_AGIOS) -endif() -message(STATUS "[gekkofs] AGIOS scheduling: ${ENABLE_AGIOS}") option(GKFS_ENABLE_FORWARDING "Enable forwarding mode" OFF) option(GKFS_ENABLE_AGIOS "Enable AGIOS scheduling library" OFF) diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 39ddf5cfe..a4c5507bc 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -531,11 +531,7 @@ ssize_t gkfs_read(int fd, void* buf, size_t count) { ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) { auto file = CTX->file_map()->get(fd); -<<<<<<< Updated upstream auto pos = offset; // keep track of current position -======= - auto pos = offset; // keep truck of current position ->>>>>>> Stashed changes ssize_t read = 0; ssize_t ret; for (int i = 0; i < iovcnt; ++i) { diff --git a/src/client/preload.cpp b/src/client/preload.cpp index e29a78e87..8f37af8fc 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -36,7 +36,6 @@ namespace { // make sure that things are only initialized once pthread_once_t init_env_thread = PTHREAD_ONCE_INIT; -<<<<<<< Updated upstream #ifdef GKFS_ENABLE_FORWARDING pthread_t mapper; bool forwarding_running; @@ -44,10 +43,6 @@ bool forwarding_running; pthread_mutex_t remap_mutex; pthread_cond_t remap_signal; #endif -======= -pthread_t mapper; -bool forwarding_running; ->>>>>>> Stashed changes inline void exit_error_msg(int errcode, const string& msg) { @@ -137,7 +132,6 @@ void init_ld_environment_() { } #ifdef GKFS_ENABLE_FORWARDING -<<<<<<< Updated upstream void *forwarding_mapper(void* p) { struct timespec timeout; clock_gettime(CLOCK_REALTIME, &timeout); @@ -145,14 +139,10 @@ void *forwarding_mapper(void* p) { int previous = -1; -======= -void *forwarding_mapper(void *p) { ->>>>>>> Stashed changes while (forwarding_running) { try { gkfs::util::load_forwarding_map(); -<<<<<<< Updated upstream if (previous != CTX->fwd_host_id()) { LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); @@ -172,25 +162,11 @@ void *forwarding_mapper(void *p) { #endif #ifdef GKFS_ENABLE_FORWARDING -======= - LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); - } catch (std::exception& e){ - exit_error_msg(EXIT_FAILURE, fmt::format("Unable set the forwarding host '{}'", e.what())); - } - - // Sleeps for 10 seconds - sleep(10); - } -} -#endif - ->>>>>>> Stashed changes void init_forwarding_mapper() { forwarding_running = true; pthread_create(&mapper, NULL, forwarding_mapper, NULL); } -<<<<<<< Updated upstream #endif #ifdef GKFS_ENABLE_FORWARDING @@ -202,14 +178,6 @@ void destroy_forwarding_mapper() { pthread_join(mapper, NULL); } #endif -======= - -void destroy_forwarding_mapper() { - forwarding_running = false; - - pthread_join(mapper, NULL); -} ->>>>>>> Stashed changes void log_prog_name() { std::string line; @@ -270,13 +238,9 @@ void init_preload() { CTX->unprotect_user_fds(); -<<<<<<< Updated upstream #ifdef GKFS_ENABLE_FORWARDING init_forwarding_mapper(); #endif -======= - init_forwarding_mapper(); ->>>>>>> Stashed changes gkfs::preload::start_interception(); } @@ -285,13 +249,9 @@ void init_preload() { * Called last when preload library is used with the LD_PRELOAD environment variable */ void destroy_preload() { -<<<<<<< Updated upstream #ifdef GKFS_ENABLE_FORWARDING destroy_forwarding_mapper(); #endif -======= - destroy_forwarding_mapper(); ->>>>>>> Stashed changes CTX->clear_hosts(); LOG(DEBUG, "Peer information deleted"); diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 8e14d17af..52345e3ce 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -12,7 +12,6 @@ set(DAEMON_SRC handler/srv_metadata.cpp handler/srv_data.cpp handler/srv_management.cpp - scheduler/agios.cpp ) set(DAEMON_HEADERS ../../include/config.hpp @@ -24,7 +23,6 @@ set(DAEMON_HEADERS ../../include/global/path_util.hpp ../../include/daemon/daemon.hpp ../../include/daemon/util.hpp - ../../include/daemon/scheduler/agios.hpp ../../include/daemon/ops/metadentry.hpp ../../include/daemon/classes/fs_data.hpp ../../include/daemon/classes/rpc_data.hpp @@ -42,7 +40,6 @@ target_link_libraries(gkfs_daemon env_util spdlog fmt::fmt - ${AGIOS_LIBRARIES} # margo libs ${ABT_LIBRARIES} mercury @@ -56,7 +53,6 @@ target_link_libraries(gkfs_daemon target_include_directories(gkfs_daemon PRIVATE - ${AGIOS_INCLUDE_DIRS} ${ABT_INCLUDE_DIRS} ${MARGO_INCLUDE_DIRS} ) @@ -147,4 +143,4 @@ if(GKFS_ENABLE_FORWARDING) install(TARGETS gkfwd_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) -endif() \ No newline at end of file +endif() diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index c223436cc..4eae87c9b 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -644,19 +644,11 @@ static hg_return_t rpc_srv_get_chunk_stat(hg_handle_t handle) { DEFINE_MARGO_RPC_HANDLER(rpc_srv_get_chunk_stat) #ifdef GKFS_ENABLE_AGIOS -<<<<<<< Updated upstream void *agios_eventual_callback(int64_t request_id, void* info) { GKFS_DATA->spdlogger()->debug("{}() custom callback request {} is ready", __func__, request_id); ABT_eventual_set((ABT_eventual) info, &request_id, sizeof(int64_t)); -======= -void *agios_eventual_callback(int64_t request_id, void *info) { - GKFS_DATA->spdlogger()->debug("{}() custom callback request {} is ready", __func__, request_id); - - ABT_eventual_set((ABT_eventual) info, &request_id, sizeof(int64_t)); - ->>>>>>> Stashed changes return 0; } #endif -- GitLab From f734de04c5e1083e2b1ecfd29e083428ace307b4 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 17 Aug 2020 11:38:51 -0300 Subject: [PATCH 69/71] Remove unecessary command from docker --- docker/build_env.docker | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/build_env.docker b/docker/build_env.docker index ef83cbaa9..346807b86 100644 --- a/docker/build_env.docker +++ b/docker/build_env.docker @@ -80,7 +80,6 @@ RUN yum -y -q update && yum -y -q install \ automake \ cmake \ cmake3 \ - sudo \ gcc \ gcc-c++ \ openssl-devel \ -- GitLab From a92582a58fac35eec9b5384f19b6590bbc30cfc6 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 17 Aug 2020 14:44:29 -0300 Subject: [PATCH 70/71] Merge the code with the latest updates --- src/daemon/CMakeLists.txt | 5 +++-- src/daemon/handler/srv_data.cpp | 6 +++--- tests/integration/conftest.py | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index b9adc6c8c..ad35b89e8 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -63,7 +63,6 @@ install(TARGETS gkfs_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) - if(GKFS_ENABLE_FORWARDING) set(FWD_DAEMON_SRC ../global/rpc/rpc_util.cpp @@ -71,10 +70,11 @@ if(GKFS_ENABLE_FORWARDING) daemon.cpp util.cpp ops/metadentry.cpp + ops/data.cpp classes/fs_data.cpp classes/rpc_data.cpp - handler/srv_metadata.cpp handler/srv_data.cpp + handler/srv_metadata.cpp handler/srv_management.cpp scheduler/agios.cpp ) @@ -89,6 +89,7 @@ if(GKFS_ENABLE_FORWARDING) ../../include/daemon/daemon.hpp ../../include/daemon/util.hpp ../../include/daemon/scheduler/agios.hpp + ../../include/daemon/ops/data.hpp ../../include/daemon/ops/metadentry.hpp ../../include/daemon/classes/fs_data.hpp ../../include/daemon/classes/rpc_data.hpp diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index 6c0687d6d..27863fd4b 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -56,9 +56,9 @@ static hg_return_t rpc_srv_write(hg_handle_t handle) { auto hgi = margo_get_info(handle); auto mid = margo_hg_info_get_instance(hgi); auto bulk_size = margo_bulk_get_size(in.bulk_handle); - - GKFS_DATA->spdlogger()->debug("{}() path: {}, size: {}, offset: {}", __func__, - in.path, bulk_size, in.offset); + GKFS_DATA->spdlogger()->debug( + "{}() path: '{}' chunk_start '{}' chunk_end '{}' chunk_n '{}' total_chunk_size '{}' bulk_size: '{}' offset: '{}'", + __func__, in.path, in.chunk_start, in.chunk_end, in.chunk_n, in.total_chunk_size, bulk_size, in.offset); #ifdef GKFS_ENABLE_AGIOS int *data; ABT_eventual eventual = ABT_EVENTUAL_NULL; diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 44113e353..4dd28b8d6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -92,7 +92,7 @@ def gkfs_daemon(test_workspace, request): daemon.shutdown() @pytest.fixture -def gkfs_client(test_workspace, request): +def gkfs_client(test_workspace): """ Sets up a gekkofs client environment so that operations (system calls, library calls, ...) can @@ -102,7 +102,7 @@ def gkfs_client(test_workspace, request): return Client(test_workspace) @pytest.fixture -def gkfs_shell(test_workspace, request): +def gkfs_shell(test_workspace): """ Sets up a gekkofs environment so that shell commands (stat, ls, mkdir, etc.) can be issued to a co-running daemon. @@ -131,7 +131,7 @@ def gkfwd_daemon_factory(test_workspace, request): return FwdDaemonCreator(interface, test_workspace) @pytest.fixture -def gkfwd_client_factory(test_workspace, request): +def gkfwd_client_factory(test_workspace): """ Sets up a gekkofs client environment so that operations (system calls, library calls, ...) can -- GitLab From 95916d10c555386692d9f492570d6db142f0e73f Mon Sep 17 00:00:00 2001 From: Jean Bez Date: Wed, 9 Sep 2020 14:31:29 +0000 Subject: [PATCH 71/71] Update .gitlab-ci.yml --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9e69feb0c..e379d82ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,9 +23,7 @@ variables: LIBGKFS_LOG_OUTPUT: "${CI_PROJECT_DIR}/logs/gkfs_client.log" GIT_SUBMODULE_STRATEGY: recursive -# Temporary new image file -# image: gekkofs/gekkofs:build_env-0.8.0 -image: jeanbez/gekkofs-forwarding:latest +image: gekkofs/gekkofs:build_env-0.8.0 compile dependencies: stage: build deps -- GitLab