From 8d69ae7ab7adfd0d1b01161379ea537d117d538b Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Tue, 31 Mar 2020 14:27:34 +0200 Subject: [PATCH 1/4] Adding verbs support to scripts, Add mogon1 preset --- scripts/compile_dep.sh | 24 ++++++++++++++++++++---- scripts/dl_dep.sh | 27 +++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 9b8335ab6..1a37a58a7 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -9,7 +9,12 @@ SOURCE="" INSTALL="" DEP_CONFIG="" -VALID_DEP_OPTIONS="mogon2 direct all" +VALID_DEP_OPTIONS="mogon2 mogon1 direct all" + +MOGON1_DEPS=( + "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" "date" "verbs" +) MOGON2_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" @@ -52,7 +57,7 @@ optional arguments: defaults to 'all' -c , --config allows additional configurations, e.g., for specific clusters - supported values: {mogon2, direct, all} + supported values: {mogon1, mogon2, direct, all} defaults to 'direct' -d , --dependency download a specific dependency and ignore --config setting. If unspecified @@ -69,6 +74,10 @@ list_dependencies() { echo "Available dependencies: " + echo -n " Mogon 1: " + for d in "${MOGON1_DEPS[@]}"; do + echo -n "$d " + done echo -n " Mogon 2: " for d in "${MOGON2_DEPS[@]}"; do echo -n "$d " @@ -94,7 +103,6 @@ check_dependency() { if echo "${DEPENDENCY}" | grep -q "${DEP}"; then return fi -# [[ "${DEPENDENCY}" == "${DEP}" ]] && return else # if not check if dependency is part of dependency config for e in "${DEP_CONFIG[@]}"; do @@ -217,6 +225,10 @@ else fi # enable predefined dependency template case ${TMP_DEP_CONF} in +mogon1) + DEP_CONFIG=("${MOGON1_DEPS[@]}") + echo "'Mogon1' dependencies are compiled" + ;; mogon2) DEP_CONFIG=("${MOGON2_DEPS[@]}") echo "'Mogon2' dependencies are compiled" @@ -320,7 +332,11 @@ if check_dependency "ofi" "${DEP_CONFIG[@]}"; then CURR=${SOURCE}/libfabric prepare_build_dir ${CURR} cd ${CURR}/build - ../configure --prefix=${INSTALL} --enable-tcp=yes + OFI_CONFIG="../configure --prefix=${INSTALL} --enable-tcp=yes" + if check_dependency "verbs" "${DEP_CONFIG[@]}"; then + OFI_CONFIG="${OFI_CONFIG} --enable-verbs=yes" + fi + ${OFI_CONFIG} make -j${CORES} make install [ "${PERFORM_TEST}" ] && make check diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 9cb685592..65d7453b4 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -9,7 +9,12 @@ NA_LAYER="" DEP_CONFIG="" VERBOSE=false -VALID_DEP_OPTIONS="mogon2 direct all" +VALID_DEP_OPTIONS="mogon2 mogon1 direct all" + +MOGON1_DEPS=( + "zstd" "lz4" "snappy" "capstone" "ofi-verbs" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" "date" +) MOGON2_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" @@ -48,6 +53,10 @@ list_dependencies() { echo "Available dependencies: " + echo -n " Mogon 1: " + for d in "${MOGON1_DEPS[@]}"; do + echo -n "$d " + done echo -n " Mogon 2: " for d in "${MOGON2_DEPS[@]}"; do echo -n "$d " @@ -248,6 +257,10 @@ fi # enable predefined dependency template case ${TMP_DEP_CONF} in +mogon1) + DEP_CONFIG=("${MOGON1_DEPS[@]}") + [[ -z "${DEPENDENCY}" ]] && echo "'Mogon1' dependencies are downloaded" + ;; mogon2) DEP_CONFIG=("${MOGON2_DEPS[@]}") [[ -z "${DEPENDENCY}" ]] && echo "'Mogon2' dependencies are downloaded" @@ -308,8 +321,14 @@ if check_dependency "bmi" "${DEP_CONFIG[@]}"; then fi # get libfabric -if check_dependency "ofi" "${DEP_CONFIG[@]}"; then - if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then +if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then + if check_dependency "ofi-experimental" "${DEP_CONFIG[@]}"; then + wgetdeps "libfabric" "https://github.com/ofiwg/libfabric/releases/download/v1.9.1/libfabric-1.9.1.tar.bz2" & + elif check_dependency "ofi-verbs" "${DEP_CONFIG[@]}"; then + # libibverbs 1.2.1-1 used on mogon 1i (installed on system) which is linked to libfabric + # libfabric 1.8 random RPCs fail to be send. 1.9 RPC client cannot be started when in an MPI environment + wgetdeps "libfabric" "https://github.com/ofiwg/libfabric/releases/download/v1.7.2/libfabric-1.7.2.tar.gz" & + elif check_dependency "ofi" "${DEP_CONFIG[@]}"; then wgetdeps "libfabric" "https://github.com/ofiwg/libfabric/releases/download/v1.8.1/libfabric-1.8.1.tar.bz2" & fi fi @@ -345,4 +364,4 @@ if check_dependency "date" "${DEP_CONFIG[@]}"; then fi # Wait for all download to be completed wait -echo "Done" \ No newline at end of file +echo "Done" -- GitLab From 335edd6a93a362bec90f26796000c7afe232c212 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Tue, 31 Mar 2020 14:28:02 +0200 Subject: [PATCH 2/4] Updating hermes git submodule --- external/hermes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/hermes b/external/hermes index 31d15c233..a25a82d40 160000 --- a/external/hermes +++ b/external/hermes @@ -1 +1 @@ -Subproject commit 31d15c2339d48dba6561546a45ecd759c16d8455 +Subproject commit a25a82d401b4a0d986088c6802a08fd79185a278 -- GitLab From a32afa92624d2e63398e4747c5e4fdef1fa22320 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Tue, 31 Mar 2020 14:29:12 +0200 Subject: [PATCH 3/4] Adding ofi+verbs support to gkfs daemon --- include/global/global_defs.hpp | 1 + src/daemon/daemon.cpp | 31 ++++++++++++++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/include/global/global_defs.hpp b/include/global/global_defs.hpp index 8bd958615..ffe78f41f 100644 --- a/include/global/global_defs.hpp +++ b/include/global/global_defs.hpp @@ -42,6 +42,7 @@ namespace protocol { constexpr auto ofi_psm2 = "ofi+psm2"; constexpr auto ofi_sockets = "ofi+sockets"; constexpr auto ofi_tcp = "ofi+tcp"; +constexpr auto ofi_verbs = "ofi+verbs"; } // namespace protocol } // namespace rpc diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 9fbf1d3f7..5a709f0e3 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -32,9 +32,13 @@ #include #include #include -#include #include +extern "C" { +#include +#include +} + using namespace std; namespace po = boost::program_options; namespace bfs = boost::filesystem; @@ -279,11 +283,14 @@ int main(int argc, const char* argv[]) { ("mountdir,m", po::value()->required(), "User Fuse mountdir") ("rootdir,r", po::value()->required(), "data directory") ("metadir,i", po::value(), "metadata directory, if not set rootdir is used for metadata ") - ("listen,l", po::value(), "Address or interface to bind the daemon on. Default: local hostname") + ("listen,l", po::value(), "Address or interface to bind the daemon on. Default: local hostname.\n" + "When used with ofi+verbs the FI_VERBS_IFACE environment variable is set accordingly " + "which associates the verbs device with the network interface. In case FI_VERBS_IFACE " + "is already defined, the argument is ignored. Default 'ib'.") ("hosts-file,H", po::value(), "Shared file used by deamons to register their " "enpoints. (default './gkfs_hosts.txt')") - ("version,h", "print version and exit"); + ("version", "print version and exit"); po::variables_map vm; po::store(po::parse_command_line(argc, argv, desc), vm); @@ -324,12 +331,22 @@ int main(int argc, const char* argv[]) { initialize_loggers(); GKFS_DATA->spdlogger(spdlog::get("main")); - - string addr; + string addr{}; if (vm.count("listen")) { addr = vm["listen"].as(); + // ofi+verbs requires an empty addr to bind to the ib interface + if (RPC_PROTOCOL == string(gkfs::rpc::protocol::ofi_verbs)) { + /* + * FI_VERBS_IFACE : The prefix or the full name of the network interface associated with the verbs device (default: ib) + * Mercury does not allow to bind to an address when ofi+verbs is used + */ + if (!secure_getenv("FI_VERBS_IFACE")) + setenv("FI_VERBS_IFACE", addr.c_str(), 1); + addr = ""s; + } } else { - addr = get_my_hostname(true); + if (RPC_PROTOCOL != string(gkfs::rpc::protocol::ofi_verbs)) + addr = get_my_hostname(true); } GKFS_DATA->bind_addr(fmt::format("{}://{}", RPC_PROTOCOL, addr)); @@ -389,4 +406,4 @@ int main(int argc, const char* argv[]) { destroy_enviroment(); GKFS_DATA->spdlogger()->info("{}() Complete. Exiting...", __func__); return 0; -} \ No newline at end of file +} -- GitLab From f6a49aa2d640c091a1a144349f34b0320ceaa386 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Tue, 31 Mar 2020 14:29:50 +0200 Subject: [PATCH 4/4] Adding (experimental) RocksDB version 6.7.3 for testing --- scripts/dl_dep.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 65d7453b4..ca859984b 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -351,6 +351,8 @@ fi # get rocksdb if check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz" & +elif check_dependency "rocksdb-experimental" "${DEP_CONFIG[@]}"; then + wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.7.3.tar.gz" & fi # get syscall_intercept -- GitLab