From 4d75b40ff3f1ad991674bb6069aa6a8657c48b2a Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Fri, 24 Jul 2020 13:22:24 +0200 Subject: [PATCH 1/4] Updating scripts for psm2. Version bumps to Mercury, Margo to March 2020 --- external/hermes | 2 +- scripts/compile_dep.sh | 6 +++++- scripts/dl_dep.sh | 9 ++++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/external/hermes b/external/hermes index a25a82d40..0af45bfa6 160000 --- a/external/hermes +++ b/external/hermes @@ -1 +1 @@ -Subproject commit a25a82d401b4a0d986088c6802a08fd79185a278 +Subproject commit 0af45bfa667f7ff9c78167ef94d975bffbd879f0 diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 51d7e94b4..1b1dd474e 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -18,7 +18,7 @@ MOGON1_DEPS=( MOGON2_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "syscall_intercept" "date" "psm2" ) DIRECT_DEPS=( @@ -337,6 +337,10 @@ if check_dependency "ofi" "${DEP_CONFIG[@]}"; then OFI_CONFIG="../configure --prefix=${INSTALL} --enable-tcp=yes" if check_dependency "verbs" "${DEP_CONFIG[@]}"; then OFI_CONFIG="${OFI_CONFIG} --enable-verbs=yes" + elif check_dependency "psm2" "${DEP_CONFIG[@]}"; then + OFI_CONFIG="${OFI_CONFIG} --enable-psm2=yes --with-psm2-src=${SOURCE}/psm2" + elif check_dependency "psm2-system" "${DEP_CONFIG[@]}"; then + OFI_CONFIG="${OFI_CONFIG} --enable-psm2=yes" fi ${OFI_CONFIG} make -j${CORES} diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 60bd2b456..20ea152bc 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -82,7 +82,6 @@ check_dependency() { if echo "${DEPENDENCY}" | grep -q "${DEP}"; then return fi -# [[ "${DEPENDENCY}" == "${DEP}" ]] && return else # if not check if dependency is part of dependency config for e in "${DEP_CONFIG[@]}"; do @@ -333,9 +332,13 @@ if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then fi fi +if check_dependency "psm2" "${DEP_CONFIG[@]}"; then + wgetdeps "psm2" "https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz" & +fi + # get Mercury if check_dependency "mercury" "${DEP_CONFIG[@]}"; then - clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "fd410dfb9852b2b98d21113531f3058f45bfcd64" "--recurse-submodules" & + clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "41caa143a07ed179a3149cac4af0dc7aa3f946fd" "--recurse-submodules" & fi # get Argobots @@ -345,7 +348,7 @@ fi # get Margo if check_dependency "margo" "${DEP_CONFIG[@]}"; then - clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "016dbdce22da3fe4f97b46c20a53bced9370a217" & + clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "v0.6.3" & fi # get rocksdb -- GitLab From 0d0222da51ff14de97809f97db864cac8d5757ee Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 2 Apr 2020 18:05:15 +0200 Subject: [PATCH 2/4] psm2 support for daemon --- src/daemon/daemon.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 5a709f0e3..a7a920627 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -103,8 +103,7 @@ void init_rpc_server(const string& protocol_port) { hg_addr_t addr_self; hg_size_t addr_self_cstring_sz = 128; char addr_self_cstring[128]; - // IMPORTANT: this struct needs to be zeroed before use - struct hg_init_info hg_options = {}; + struct hg_init_info hg_options = HG_INIT_INFO_INITIALIZER; #if USE_SHM hg_options.auto_sm = HG_TRUE; #else @@ -112,6 +111,8 @@ void init_rpc_server(const string& protocol_port) { #endif hg_options.stats = HG_FALSE; hg_options.na_class = nullptr; + if (gkfs::rpc::protocol::ofi_psm2 == string(RPC_PROTOCOL)) + hg_options.na_init_info.progress_mode = NA_NO_BLOCK; // Start Margo (this will also initialize Argobots and Mercury internally) auto mid = margo_init_opt(protocol_port.c_str(), MARGO_SERVER_MODE, -- GitLab From 70fd346c3faa448b607b812d5913a1b972336653 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 23 Jul 2020 15:42:38 +0200 Subject: [PATCH 3/4] Force no blocking progress for psm2 on client --- src/client/preload.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client/preload.cpp b/src/client/preload.cpp index ad09cff32..a48c36c25 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -19,6 +19,7 @@ #include #include +#include #include @@ -64,6 +65,9 @@ bool init_hermes_client(const std::string& transport_prefix) { #if USE_SHM opts |= hermes::use_auto_sm; #endif + if (gkfs::rpc::protocol::ofi_psm2 == string(RPC_PROTOCOL)) { + opts |= hermes::force_no_block_progress; + } ld_network_service = std::make_unique( -- GitLab From 7f037994ed73ee9a09a61bb6428117f3adc81600 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Mon, 15 Jun 2020 13:34:13 +0100 Subject: [PATCH 4/4] Added ngio deps set --- scripts/compile_dep.sh | 24 +++++++++++++++++++++--- scripts/dl_dep.sh | 23 ++++++++++++++++++++--- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 1b1dd474e..79a0ad692 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -9,7 +9,7 @@ SOURCE="" INSTALL="" DEP_CONFIG="" -VALID_DEP_OPTIONS="mogon2 mogon1 direct all" +VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all" MOGON1_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" @@ -21,6 +21,11 @@ MOGON2_DEPS=( "syscall_intercept" "date" "psm2" ) +NGIO_DEPS=( + "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" "date" "psm2" +) + DIRECT_DEPS=( "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" ) @@ -57,7 +62,7 @@ optional arguments: defaults to 'all' -c , --config allows additional configurations, e.g., for specific clusters - supported values: {mogon1, mogon2, direct, all} + supported values: {mogon1, mogon2, ngio, direct, all} defaults to 'direct' -d , --dependency download a specific dependency and ignore --config setting. If unspecified @@ -78,19 +83,27 @@ list_dependencies() { for d in "${MOGON1_DEPS[@]}"; do echo -n "$d " done + echo echo -n " Mogon 2: " for d in "${MOGON2_DEPS[@]}"; do echo -n "$d " done + echo + echo -n " NGIO: " + for d in "${NGIO_DEPS[@]}"; do + echo -n "$d " + done + echo echo -n " Direct GekkoFS dependencies: " for d in "${DIRECT_DEPS[@]}"; do echo -n "$d " done + echo echo -n " All: " for d in "${ALL_DEPS[@]}"; do echo -n "$d " done - echo "" + echo } check_dependency() { @@ -233,6 +246,10 @@ mogon2) DEP_CONFIG=("${MOGON2_DEPS[@]}") echo "'Mogon2' dependencies are compiled" ;; +ngio) + DEP_CONFIG=("${NGIO_DEPS[@]}") + echo "'NGIO' dependencies are compiled" + ;; all) DEP_CONFIG=("${ALL_DEPS[@]}") echo "'All' dependencies are compiled" @@ -260,6 +277,7 @@ set -e export CPATH="${CPATH}:${INSTALL}/include" export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL}/lib:${INSTALL}/lib64" +export PKG_CONFIG_PATH="${INSTALL}/lib/pkgconfig:${PKG_CONFIG_PATH}" ## Third party dependencies diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 20ea152bc..d8cbff6a7 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -9,7 +9,7 @@ NA_LAYER="" DEP_CONFIG="" VERBOSE=false -VALID_DEP_OPTIONS="mogon2 mogon1 direct all" +VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all" MOGON1_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi-verbs" "mercury" "argobots" "margo" "rocksdb" @@ -21,6 +21,11 @@ MOGON2_DEPS=( "syscall_intercept-glibc3" "date" "psm2" ) +NGIO_DEPS=( + "zstd" "lz4" "snappy" "capstone" "ofi-experimental" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" "date" "psm2" + +) DIRECT_DEPS=( "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" ) @@ -57,19 +62,27 @@ list_dependencies() { for d in "${MOGON1_DEPS[@]}"; do echo -n "$d " done + echo echo -n " Mogon 2: " for d in "${MOGON2_DEPS[@]}"; do echo -n "$d " done + echo + echo -n " NGIO: " + for d in "${NGIO_DEPS[@]}"; do + echo -n "$d " + done + echo echo -n " Direct GekkoFS dependencies: " for d in "${DIRECT_DEPS[@]}"; do echo -n "$d " done + echo echo -n " All: " for d in "${ALL_DEPS[@]}"; do echo -n "$d " done - echo "" + echo } check_dependency() { @@ -178,7 +191,7 @@ optional arguments: defaults to 'ofi' -c , --config allows additional configurations, e.g., for specific clusters - supported values: {mogon2, direct, all} + supported values: {mogon2, mogon1, ngio, direct, all} defaults to 'direct' -d , --dependency download a specific dependency and ignore --config setting. If unspecified @@ -264,6 +277,10 @@ mogon2) DEP_CONFIG=("${MOGON2_DEPS[@]}") [[ -z "${DEPENDENCY}" ]] && echo "'Mogon2' dependencies are downloaded" ;; +ngio) + DEP_CONFIG=("${NGIO_DEPS[@]}") + [[ -z "${DEPENDENCY}" ]] && echo "'NGIO' dependencies are downloaded" + ;; all) DEP_CONFIG=("${ALL_DEPS[@]}") [[ -z "${DEPENDENCY}" ]] && echo "'All' dependencies are downloaded" -- GitLab