From 0bc9b21a8cfad507f5e0889d1d4dd8f139a1f590 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Wed, 10 Mar 2021 12:57:55 +0100 Subject: [PATCH 01/16] General version bump --- scripts/compile_dep.sh | 11 +++++++++++ scripts/dl_dep.sh | 21 +++++++++++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index d187b8115..362b0e0b0 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -372,6 +372,17 @@ if check_dependency "capstone" "${DEP_CONFIG[@]}"; then make -j"${CORES}" install fi +# build json-c for Margo +if check_dependency "json-c" "${DEP_CONFIG[@]}"; then + # XXX NOT WORKING + echo "############################################################ Installing: json-c" + CURR=${SOURCE}/json-c + prepare_build_dir ${CURR} + cd ${CURR}/build + ../configure --prefix=${INSTALL} + make -j${CORES} install +fi + # build BMI if check_dependency "bmi" "${DEP_CONFIG[@]}"; then if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "all" ]]; then diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 9b4e90f34..6a10688ff 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -39,7 +39,7 @@ VERBOSE=false VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all ci" MOGON1_DEPS=( - "zstd" "lz4" "snappy" "capstone" "ofi-verbs" "mercury" "argobots" "margo" "rocksdb" + "zstd" "lz4" "snappy" "capstone" "ofi-experimental" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" ) @@ -372,6 +372,11 @@ if check_dependency "capstone" "${DEP_CONFIG[@]}"; then wgetdeps "capstone" "https://github.com/aquynh/capstone/archive/4.0.1.tar.gz" & fi +# get json-c for Margo +if check_dependency "json-c" "${DEP_CONFIG[@]}"; then + wgetdeps "json-c" "https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz" & +fi + ## Direct GekkoFS dependencies # get BMI @@ -384,7 +389,7 @@ fi # get libfabric if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then if check_dependency "ofi-experimental" "${DEP_CONFIG[@]}"; then - clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.9.1" & + clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.11.1" & elif check_dependency "ofi-verbs" "${DEP_CONFIG[@]}"; then # libibverbs 1.2.1-1 used on mogon 1i (installed on system) which is linked to libfabric # libfabric 1.8 random RPCs fail to be send. 1.9 RPC client cannot be started when in an MPI environment @@ -395,29 +400,29 @@ if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then fi if check_dependency "psm2" "${DEP_CONFIG[@]}"; then - wgetdeps "psm2" "https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz" & + wgetdeps "psm2" "https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz" & fi # get Mercury if check_dependency "mercury" "${DEP_CONFIG[@]}"; then - clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "41caa143a07ed179a3149cac4af0dc7aa3f946fd" "--recurse-submodules" & + clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "3b2934490fbbcc291541e9f417fed120ffc1a04f" "--recurse-submodules" & fi # get Argobots if check_dependency "argobots" "${DEP_CONFIG[@]}"; then - wgetdeps "argobots" "https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz" & + wgetdeps "argobots" "https://github.com/pmodels/argobots/archive/v1.0.1.tar.gz" & fi # get Margo if check_dependency "margo" "${DEP_CONFIG[@]}"; then - clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "v0.6.3" & + clonedeps "margo" "https://github.com/mochi-hpc/mochi-margo" "084b61323933a0e014653eecf8f6b66a94196a2d" & fi # get rocksdb if check_dependency "rocksdb-experimental" "${DEP_CONFIG[@]}"; then - wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz" & + wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.15.5.tar.gz" & elif check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then - wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz" & + wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz" & fi # get syscall_intercept -- GitLab From d16629b3d835f3e7cab4a62337492b5faf77c3d4 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Mon, 19 Apr 2021 13:02:37 +0200 Subject: [PATCH 02/16] deps version bump --- scripts/compile_dep.sh | 3 ++- scripts/dl_dep.sh | 17 ++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 362b0e0b0..3d6113640 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -502,7 +502,8 @@ if check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then cd "${CURR}" make clean USE_RTTI=1 make -j"${CORES}" static_lib - INSTALL_PATH="${INSTALL}" make install + # INSTALL_PATH older version, PREFIX newer versions + INSTALL_PATH="${INSTALL}" PREFIX="${INSTALL}" make install fi # syscall_intercept diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 6a10688ff..5d69a6f76 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -389,7 +389,7 @@ fi # get libfabric if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then if check_dependency "ofi-experimental" "${DEP_CONFIG[@]}"; then - clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.11.1" & + clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.12.1" & elif check_dependency "ofi-verbs" "${DEP_CONFIG[@]}"; then # libibverbs 1.2.1-1 used on mogon 1i (installed on system) which is linked to libfabric # libfabric 1.8 random RPCs fail to be send. 1.9 RPC client cannot be started when in an MPI environment @@ -404,30 +404,33 @@ if check_dependency "psm2" "${DEP_CONFIG[@]}"; then fi # get Mercury -if check_dependency "mercury" "${DEP_CONFIG[@]}"; then - clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "3b2934490fbbcc291541e9f417fed120ffc1a04f" "--recurse-submodules" & +if check_dependency "mercury-experimental" "${DEP_CONFIG[@]}"; then + # currently not compatible with Hermes + clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "71672b0d453dc5a64d20f46b86f25fed12566eff" "--recurse-submodules" & +elif check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then + clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "4796aeeb90ba58be8a3b17a73f27aa3afed1ee0f" "--recurse-submodules" & fi # get Argobots if check_dependency "argobots" "${DEP_CONFIG[@]}"; then - wgetdeps "argobots" "https://github.com/pmodels/argobots/archive/v1.0.1.tar.gz" & + wgetdeps "argobots" "https://github.com/pmodels/argobots/archive/refs/tags/v1.1.tar.gz" & fi # get Margo if check_dependency "margo" "${DEP_CONFIG[@]}"; then - clonedeps "margo" "https://github.com/mochi-hpc/mochi-margo" "084b61323933a0e014653eecf8f6b66a94196a2d" & + clonedeps "margo" "https://github.com/mochi-hpc/mochi-margo" "84c7a123fcabaac1545cc0352a74ea27ed1d6a83" & fi # get rocksdb if check_dependency "rocksdb-experimental" "${DEP_CONFIG[@]}"; then - wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.15.5.tar.gz" & + wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/refs/tags/v6.16.4.tar.gz" & elif check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz" & fi # get syscall_intercept if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then - clonedeps "syscall_intercept" "https://github.com/pmem/syscall_intercept.git" "f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" "" "syscall_intercept.patch" & + clonedeps "syscall_intercept" "https://github.com/pmem/syscall_intercept.git" "304404581c57d43478438d175099d20260bae74e" "" "syscall_intercept.patch" & fi # get AGIOS -- GitLab From 78d9e0da449632f97a1bbfea9d31910512a7c367 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Mon, 19 Apr 2021 13:03:20 +0200 Subject: [PATCH 03/16] reformating --- scripts/compile_dep.sh | 100 ++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 3d6113640..c0794af9e 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -54,12 +54,12 @@ NGIO_DEPS=( ) DIRECT_DEPS=( - "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" ) ALL_DEPS=( "bzip2" "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" "agios" + "syscall_intercept" "date" "agios" ) CI_DEPS=( @@ -67,7 +67,7 @@ CI_DEPS=( ) usage_short() { - echo " + echo " usage: compile_dep.sh [-h] [-l] [-n ] [-c ] [-d ] [-j ] source_path install_path " @@ -114,22 +114,22 @@ list_dependencies() { for d in "${MOGON1_DEPS[@]}"; do echo -n "$d " done - echo + echo echo -n " Mogon 2: " for d in "${MOGON2_DEPS[@]}"; do echo -n "$d " done - echo + echo echo -n " NGIO: " for d in "${NGIO_DEPS[@]}"; do echo -n "$d " done - echo + echo echo -n " Direct GekkoFS dependencies: " for d in "${DIRECT_DEPS[@]}"; do echo -n "$d " done - echo + echo echo -n " All: " for d in "${ALL_DEPS[@]}"; do echo -n "$d " @@ -139,28 +139,28 @@ list_dependencies() { for d in "${CI_DEPS[@]}"; do echo -n "$d " done - echo + echo } check_dependency() { - local DEP=$1 - shift - local DEP_CONFIG=("$@") - # ignore template when specific dependency is set - if [[ -n "${DEPENDENCY}" ]]; then - # check if specific dependency was set and return from function - if echo "${DEPENDENCY}" | grep -q "${DEP}"; then - return - fi - else - # if not check if dependency is part of dependency config - for e in "${DEP_CONFIG[@]}"; do - if [[ "${DEP}" == "${e}" ]]; then - return + local DEP=$1 + shift + local DEP_CONFIG=("$@") + # ignore template when specific dependency is set + if [[ -n "${DEPENDENCY}" ]]; then + # check if specific dependency was set and return from function + if echo "${DEPENDENCY}" | grep -q "${DEP}"; then + return fi - done - fi - false + else + # if not check if dependency is part of dependency config + for e in "${DEP_CONFIG[@]}"; do + if [[ "${DEP}" == "${e}" ]]; then + return + fi + done + fi + false } prepare_build_dir() { @@ -275,29 +275,29 @@ fi # enable predefined dependency template case ${TMP_DEP_CONF} in mogon1) - DEP_CONFIG=("${MOGON1_DEPS[@]}") - echo "'Mogon1' dependencies are compiled" - ;; + DEP_CONFIG=("${MOGON1_DEPS[@]}") + echo "'Mogon1' dependencies are compiled" + ;; mogon2) - DEP_CONFIG=("${MOGON2_DEPS[@]}") - echo "'Mogon2' dependencies are compiled" - ;; + DEP_CONFIG=("${MOGON2_DEPS[@]}") + echo "'Mogon2' dependencies are compiled" + ;; ngio) - DEP_CONFIG=("${NGIO_DEPS[@]}") - echo "'NGIO' dependencies are compiled" - ;; + DEP_CONFIG=("${NGIO_DEPS[@]}") + echo "'NGIO' dependencies are compiled" + ;; all) - DEP_CONFIG=("${ALL_DEPS[@]}") - echo "'All' dependencies are compiled" - ;; + DEP_CONFIG=("${ALL_DEPS[@]}") + echo "'All' dependencies are compiled" + ;; ci) - DEP_CONFIG=("${CI_DEPS[@]}") - echo "'CI' dependencies are compiled" - ;; + DEP_CONFIG=("${CI_DEPS[@]}") + echo "'CI' dependencies are compiled" + ;; direct | *) - DEP_CONFIG=("${DIRECT_DEPS[@]}") - echo "'Direct' GekkoFS dependencies are compiled (default)" - ;; + DEP_CONFIG=("${DIRECT_DEPS[@]}") + echo "'Direct' GekkoFS dependencies are compiled (default)" + ;; esac USE_BMI="-DNA_USE_BMI:BOOL=OFF" @@ -419,7 +419,7 @@ if check_dependency "ofi" "${DEP_CONFIG[@]}"; then elif check_dependency "psm2-system" "${DEP_CONFIG[@]}"; then OFI_CONFIG="${OFI_CONFIG} --enable-psm2=yes" fi - ${OFI_CONFIG} + ${OFI_CONFIG} make -j${CORES} make install [ "${PERFORM_TEST}" ] && make check @@ -428,12 +428,12 @@ fi # AGIOS if check_dependency "agios" "${DEP_CONFIG[@]}"; then - echo "############################################################ Installing: AGIOS" - CURR=${SOURCE}/agios - prepare_build_dir "${CURR}" - cd "${CURR}"/build - $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" .. - make install + echo "############################################################ Installing: AGIOS" + CURR=${SOURCE}/agios + prepare_build_dir "${CURR}" + cd "${CURR}"/build + $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" .. + make install fi # Mercury -- GitLab From 2665829f5e77e8e9ff27fab5cd7083fca8b51297 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Mon, 19 Apr 2021 14:01:52 +0200 Subject: [PATCH 04/16] adding mercury atomic fix --- scripts/compile_dep.sh | 5 +++ scripts/patches/mercury_c11_atomic.patch | 44 ++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 scripts/patches/mercury_c11_atomic.patch diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index c0794af9e..f904ef13d 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -450,6 +450,11 @@ if check_dependency "mercury" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: Mercury" CURR=${SOURCE}/mercury prepare_build_dir "${CURR}" + cd "${CURR}" + if [[ $(cat "${CURR}"/version.txt) == "2.0.1rc3" ]]; then + echo "############ applying mercury atomic patch" + git apply "${PATCH_DIR}"/mercury_c11_atomic.patch + fi cd "${CURR}"/build PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig $CMAKE \ -DCMAKE_BUILD_TYPE:STRING=Release \ diff --git a/scripts/patches/mercury_c11_atomic.patch b/scripts/patches/mercury_c11_atomic.patch new file mode 100644 index 000000000..617242895 --- /dev/null +++ b/scripts/patches/mercury_c11_atomic.patch @@ -0,0 +1,44 @@ +diff --git a/src/util/mercury_atomic.h b/src/util/mercury_atomic.h +index 63aa54e..b27f270 100644 +--- a/src/util/mercury_atomic.h ++++ b/src/util/mercury_atomic.h +@@ -31,14 +31,34 @@ typedef OPA_int_t hg_atomic_int32_t; + typedef OPA_ptr_t hg_atomic_int64_t; /* OPA has only limited 64-bit support */ + # define HG_ATOMIC_VAR_INIT(x) OPA_PTR_T_INITIALIZER(x) + #elif defined(HG_UTIL_HAS_STDATOMIC_H) +-# include ++ ++# ifndef __cplusplus ++# include + typedef atomic_int hg_atomic_int32_t; +-# if (HG_UTIL_ATOMIC_LONG_WIDTH == 8) && !defined(__APPLE__) ++# if (HG_UTIL_ATOMIC_LONG_WIDTH == 8) && !defined(__APPLE__) + typedef atomic_long hg_atomic_int64_t; +-# else ++# else + typedef atomic_llong hg_atomic_int64_t; +-# endif +-# define HG_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) ++# endif ++# else ++# include ++# define _Atomic(X) std::atomic< X > ++typedef std::atomic hg_atomic_int32_t; ++# if (HG_UTIL_ATOMIC_LONG_WIDTH == 8) && !defined(__APPLE__) ++typedef std::atomic hg_atomic_int64_t; ++# else ++typedef std::atomic hg_atomic_int64_t; ++# endif ++# define memory_order_release std::memory_order_release ++# define memory_order_acquire std::memory_order_acquire ++# define memory_order_acq_rel std::memory_order_acq_rel ++# define atomic_thread_fence std::atomic_thread_fence ++# endif ++#define HG_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) ++ ++ ++ ++ + #elif defined(__APPLE__) + # include + typedef struct { -- GitLab From d42b3b49c403a78f4695fa856ee4c44285370145 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Mon, 19 Apr 2021 16:45:45 +0200 Subject: [PATCH 05/16] typo --- scripts/dl_dep.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 5d69a6f76..1bf13087b 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -407,7 +407,7 @@ fi if check_dependency "mercury-experimental" "${DEP_CONFIG[@]}"; then # currently not compatible with Hermes clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "71672b0d453dc5a64d20f46b86f25fed12566eff" "--recurse-submodules" & -elif check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then +elif check_dependency "mercury" "${DEP_CONFIG[@]}"; then clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "4796aeeb90ba58be8a3b17a73f27aa3afed1ee0f" "--recurse-submodules" & fi -- GitLab From f7e826cb5d50e94da23a7b360ea66dec01e76744 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Mon, 19 Apr 2021 18:28:48 +0200 Subject: [PATCH 06/16] fix wip --- scripts/compile_dep.sh | 6 +++--- scripts/dl_dep.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index f904ef13d..661e4fea4 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -45,7 +45,7 @@ MOGON1_DEPS=( MOGON2_DEPS=( "bzip2" "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" "psm2" + "syscall_intercept" "date" "psm2" "json-c" ) NGIO_DEPS=( @@ -374,13 +374,13 @@ fi # build json-c for Margo if check_dependency "json-c" "${DEP_CONFIG[@]}"; then - # XXX NOT WORKING echo "############################################################ Installing: json-c" CURR=${SOURCE}/json-c prepare_build_dir ${CURR} cd ${CURR}/build - ../configure --prefix=${INSTALL} + ../cmake-configure --prefix=${INSTALL} make -j${CORES} install + mv ${INSTALL}/lib64/pkgconfig/json-c.pc ${INSTALL}/lib/pkgconfig/ fi # build BMI diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 1bf13087b..940fb6e17 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -45,7 +45,7 @@ MOGON1_DEPS=( MOGON2_DEPS=( "bzip2" "zstd" "lz4" "snappy" "capstone" "ofi-experimental" "mercury" "argobots" "margo" "rocksdb-experimental" - "syscall_intercept" "date" "psm2" + "syscall_intercept" "date" "psm2" "json-c" ) NGIO_DEPS=( -- GitLab From 17be17afda2a65b31f2d9182c448d2469a374384 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Tue, 20 Apr 2021 10:09:39 +0200 Subject: [PATCH 07/16] mercury fix for atomic final --- scripts/patches/mercury_c11_atomic.patch | 27 ++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/scripts/patches/mercury_c11_atomic.patch b/scripts/patches/mercury_c11_atomic.patch index 617242895..39e00edba 100644 --- a/scripts/patches/mercury_c11_atomic.patch +++ b/scripts/patches/mercury_c11_atomic.patch @@ -1,8 +1,8 @@ diff --git a/src/util/mercury_atomic.h b/src/util/mercury_atomic.h -index 63aa54e..b27f270 100644 +index 63aa54e..92a0125 100644 --- a/src/util/mercury_atomic.h +++ b/src/util/mercury_atomic.h -@@ -31,14 +31,34 @@ typedef OPA_int_t hg_atomic_int32_t; +@@ -31,14 +31,31 @@ typedef OPA_int_t hg_atomic_int32_t; typedef OPA_ptr_t hg_atomic_int64_t; /* OPA has only limited 64-bit support */ # define HG_ATOMIC_VAR_INIT(x) OPA_PTR_T_INITIALIZER(x) #elif defined(HG_UTIL_HAS_STDATOMIC_H) @@ -33,12 +33,27 @@ index 63aa54e..b27f270 100644 +# define memory_order_acquire std::memory_order_acquire +# define memory_order_acq_rel std::memory_order_acq_rel +# define atomic_thread_fence std::atomic_thread_fence ++# define atomic_fetch_add_explicit std::atomic_fetch_add_explicit +# endif +#define HG_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) -+ -+ -+ -+ #elif defined(__APPLE__) # include typedef struct { +@@ -495,7 +512,7 @@ hg_atomic_incr64(hg_atomic_int64_t *ptr) + #if defined(_WIN32) + ret = InterlockedIncrementNoFence64(&ptr->value); + #elif defined(HG_UTIL_HAS_STDATOMIC_H) && !defined(HG_UTIL_HAS_OPA_PRIMITIVES_H) +- ret = atomic_fetch_add_explicit(ptr, 1, memory_order_acq_rel) + 1; ++ ret = atomic_fetch_add_explicit(ptr, 1l, memory_order_acq_rel) + 1; + #elif defined(__APPLE__) + ret = OSAtomicIncrement64(&ptr->value); + #else +@@ -517,7 +534,7 @@ hg_atomic_decr64(hg_atomic_int64_t *ptr) + #if defined(_WIN32) + ret = InterlockedDecrementNoFence64(&ptr->value); + #elif defined(HG_UTIL_HAS_STDATOMIC_H) && !defined(HG_UTIL_HAS_OPA_PRIMITIVES_H) +- ret = atomic_fetch_sub_explicit(ptr, 1, memory_order_acq_rel) - 1; ++ ret = atomic_fetch_sub_explicit(ptr, 1l, memory_order_acq_rel) - 1; + #elif defined(__APPLE__) + ret = OSAtomicDecrement64(&ptr->value); + #else -- GitLab From 17411d148da8fe3574ffbfd1261b130b8860e92f Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 22 Apr 2021 10:22:32 +0200 Subject: [PATCH 08/16] ofi+tcp support --- .gitmodules | 3 ++- external/hermes | 2 +- src/client/preload_util.cpp | 2 ++ src/daemon/daemon.cpp | 3 ++- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 30e601bf2..e38bc5542 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,7 @@ [submodule "external/hermes"] path = external/hermes - url = https://github.com/bsc-ssrg/hermes.git + url = https://github.com/marcvef/hermes.git + branch = ofi_tcp_support [submodule "external/fmt"] path = external/fmt url = https://github.com/fmtlib/fmt diff --git a/external/hermes b/external/hermes index 0af45bfa6..52d823c21 160000 --- a/external/hermes +++ b/external/hermes @@ -1 +1 @@ -Subproject commit 0af45bfa667f7ff9c78167ef94d975bffbd879f0 +Subproject commit 52d823c211c0fc9b2ec9087bea025685312d3b1c diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index 255588a8f..35104a92d 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -111,6 +111,8 @@ extract_protocol(const string& uri) { protocol = gkfs::rpc::protocol::ofi_psm2; } else if(uri.find(gkfs::rpc::protocol::ofi_verbs) != string::npos) { protocol = gkfs::rpc::protocol::ofi_verbs; + } else if(uri.find(gkfs::rpc::protocol::ofi_tcp) != string::npos) { + protocol = gkfs::rpc::protocol::ofi_tcp; } // check for shared memory protocol. Can be plain shared memory or real ofi // protocol + auto_sm diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index a8537ccdd..dcc873f9b 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -383,7 +383,8 @@ parse_input(const po::variables_map& vm) { rpc_protocol = vm["rpc-protocol"].as(); if(rpc_protocol != gkfs::rpc::protocol::ofi_verbs && rpc_protocol != gkfs::rpc::protocol::ofi_sockets && - rpc_protocol != gkfs::rpc::protocol::ofi_psm2) { + rpc_protocol != gkfs::rpc::protocol::ofi_psm2 && + rpc_protocol != gkfs::rpc::protocol::ofi_tcp) { throw runtime_error(fmt::format( "Given RPC protocol '{}' not supported. Check --help for supported protocols.", rpc_protocol)); -- GitLab From 8078669bfd37bfd530b3be81019053668eab7ede Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 22 Apr 2021 15:30:54 +0200 Subject: [PATCH 09/16] git modules fix --- .gitmodules | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index e38bc5542..e80646e29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,6 @@ [submodule "external/hermes"] path = external/hermes url = https://github.com/marcvef/hermes.git - branch = ofi_tcp_support [submodule "external/fmt"] path = external/fmt url = https://github.com/fmtlib/fmt -- GitLab From 648cd1323efcea1c26ad2c3b3100e3a5a4c83736 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 22 Apr 2021 15:34:04 +0200 Subject: [PATCH 10/16] module fix again --- external/hermes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/hermes b/external/hermes index 52d823c21..ca72cf9c3 160000 --- a/external/hermes +++ b/external/hermes @@ -1 +1 @@ -Subproject commit 52d823c211c0fc9b2ec9087bea025685312d3b1c +Subproject commit ca72cf9c33d5d7f84c689c00b5b50583d570cead -- GitLab From 207042a287825d059241f168eeb2d9ee8986434c Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Mon, 26 Apr 2021 15:56:30 +0200 Subject: [PATCH 11/16] bump mercury/margo version --- scripts/compile_dep.sh | 5 -- scripts/dl_dep.sh | 4 +- scripts/patches/mercury_c11_atomic.patch | 59 ------------------------ 3 files changed, 2 insertions(+), 66 deletions(-) delete mode 100644 scripts/patches/mercury_c11_atomic.patch diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 661e4fea4..82368f203 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -450,11 +450,6 @@ if check_dependency "mercury" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: Mercury" CURR=${SOURCE}/mercury prepare_build_dir "${CURR}" - cd "${CURR}" - if [[ $(cat "${CURR}"/version.txt) == "2.0.1rc3" ]]; then - echo "############ applying mercury atomic patch" - git apply "${PATCH_DIR}"/mercury_c11_atomic.patch - fi cd "${CURR}"/build PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig $CMAKE \ -DCMAKE_BUILD_TYPE:STRING=Release \ diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 940fb6e17..7b3b3e2f3 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -406,7 +406,7 @@ fi # get Mercury if check_dependency "mercury-experimental" "${DEP_CONFIG[@]}"; then # currently not compatible with Hermes - clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "71672b0d453dc5a64d20f46b86f25fed12566eff" "--recurse-submodules" & + clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "75cb615894699a4df5872ba9d187f37cc94330a3" "--recurse-submodules" & elif check_dependency "mercury" "${DEP_CONFIG[@]}"; then clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "4796aeeb90ba58be8a3b17a73f27aa3afed1ee0f" "--recurse-submodules" & fi @@ -418,7 +418,7 @@ fi # get Margo if check_dependency "margo" "${DEP_CONFIG[@]}"; then - clonedeps "margo" "https://github.com/mochi-hpc/mochi-margo" "84c7a123fcabaac1545cc0352a74ea27ed1d6a83" & + clonedeps "margo" "https://github.com/mochi-hpc/mochi-margo" "34bff25aa8ec9e9348ee07a455308bfe3c17d623" & fi # get rocksdb diff --git a/scripts/patches/mercury_c11_atomic.patch b/scripts/patches/mercury_c11_atomic.patch deleted file mode 100644 index 39e00edba..000000000 --- a/scripts/patches/mercury_c11_atomic.patch +++ /dev/null @@ -1,59 +0,0 @@ -diff --git a/src/util/mercury_atomic.h b/src/util/mercury_atomic.h -index 63aa54e..92a0125 100644 ---- a/src/util/mercury_atomic.h -+++ b/src/util/mercury_atomic.h -@@ -31,14 +31,31 @@ typedef OPA_int_t hg_atomic_int32_t; - typedef OPA_ptr_t hg_atomic_int64_t; /* OPA has only limited 64-bit support */ - # define HG_ATOMIC_VAR_INIT(x) OPA_PTR_T_INITIALIZER(x) - #elif defined(HG_UTIL_HAS_STDATOMIC_H) --# include -+ -+# ifndef __cplusplus -+# include - typedef atomic_int hg_atomic_int32_t; --# if (HG_UTIL_ATOMIC_LONG_WIDTH == 8) && !defined(__APPLE__) -+# if (HG_UTIL_ATOMIC_LONG_WIDTH == 8) && !defined(__APPLE__) - typedef atomic_long hg_atomic_int64_t; --# else -+# else - typedef atomic_llong hg_atomic_int64_t; --# endif --# define HG_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) -+# endif -+# else -+# include -+# define _Atomic(X) std::atomic< X > -+typedef std::atomic hg_atomic_int32_t; -+# if (HG_UTIL_ATOMIC_LONG_WIDTH == 8) && !defined(__APPLE__) -+typedef std::atomic hg_atomic_int64_t; -+# else -+typedef std::atomic hg_atomic_int64_t; -+# endif -+# define memory_order_release std::memory_order_release -+# define memory_order_acquire std::memory_order_acquire -+# define memory_order_acq_rel std::memory_order_acq_rel -+# define atomic_thread_fence std::atomic_thread_fence -+# define atomic_fetch_add_explicit std::atomic_fetch_add_explicit -+# endif -+#define HG_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) - #elif defined(__APPLE__) - # include - typedef struct { -@@ -495,7 +512,7 @@ hg_atomic_incr64(hg_atomic_int64_t *ptr) - #if defined(_WIN32) - ret = InterlockedIncrementNoFence64(&ptr->value); - #elif defined(HG_UTIL_HAS_STDATOMIC_H) && !defined(HG_UTIL_HAS_OPA_PRIMITIVES_H) -- ret = atomic_fetch_add_explicit(ptr, 1, memory_order_acq_rel) + 1; -+ ret = atomic_fetch_add_explicit(ptr, 1l, memory_order_acq_rel) + 1; - #elif defined(__APPLE__) - ret = OSAtomicIncrement64(&ptr->value); - #else -@@ -517,7 +534,7 @@ hg_atomic_decr64(hg_atomic_int64_t *ptr) - #if defined(_WIN32) - ret = InterlockedDecrementNoFence64(&ptr->value); - #elif defined(HG_UTIL_HAS_STDATOMIC_H) && !defined(HG_UTIL_HAS_OPA_PRIMITIVES_H) -- ret = atomic_fetch_sub_explicit(ptr, 1, memory_order_acq_rel) - 1; -+ ret = atomic_fetch_sub_explicit(ptr, 1l, memory_order_acq_rel) - 1; - #elif defined(__APPLE__) - ret = OSAtomicDecrement64(&ptr->value); - #else -- GitLab From 55837285688649491dae7f9d2546171dbaee89b4 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Tue, 27 Apr 2021 00:15:10 +0200 Subject: [PATCH 12/16] finalizing dependency versions --- scripts/compile_dep.sh | 8 ++++---- scripts/dl_dep.sh | 29 ++++++++++++----------------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 82368f203..1acaa7774 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -40,7 +40,7 @@ VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all ci" MOGON1_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" "verbs" + "syscall_intercept" "date" "verbs" "json-c" ) MOGON2_DEPS=( @@ -50,7 +50,7 @@ MOGON2_DEPS=( NGIO_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" "agios" "psm2" + "syscall_intercept" "date" "agios" "psm2" "json-c" ) DIRECT_DEPS=( @@ -59,11 +59,11 @@ DIRECT_DEPS=( ALL_DEPS=( "bzip2" "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" "agios" + "syscall_intercept" "date" "agios" "json-c" ) CI_DEPS=( - "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" + "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" "json-c" ) usage_short() { diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 7b3b3e2f3..fdbea188a 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -39,18 +39,18 @@ VERBOSE=false VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all ci" MOGON1_DEPS=( - "zstd" "lz4" "snappy" "capstone" "ofi-experimental" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" "date" "json-c" ) MOGON2_DEPS=( - "bzip2" "zstd" "lz4" "snappy" "capstone" "ofi-experimental" "mercury" "argobots" "margo" "rocksdb-experimental" + "bzip2" "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "psm2" "json-c" ) NGIO_DEPS=( - "zstd" "lz4" "snappy" "capstone" "ofi-experimental" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" "psm2" "agios" + "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" "date" "psm2" "agios" "json-c" ) DIRECT_DEPS=( @@ -59,11 +59,11 @@ DIRECT_DEPS=( ALL_DEPS=( "bzip2" "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" "agios" + "syscall_intercept" "date" "agios" "json-c" ) CI_DEPS=( - "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" + "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" "json-c" ) # Stop all backround jobs on interruption. @@ -388,14 +388,14 @@ fi # get libfabric if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then - if check_dependency "ofi-experimental" "${DEP_CONFIG[@]}"; then - clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.12.1" & + if check_dependency "ofi-legacy" "${DEP_CONFIG[@]}"; then + clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.8.1" & elif check_dependency "ofi-verbs" "${DEP_CONFIG[@]}"; then # libibverbs 1.2.1-1 used on mogon 1i (installed on system) which is linked to libfabric # libfabric 1.8 random RPCs fail to be send. 1.9 RPC client cannot be started when in an MPI environment clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.7.2" & elif check_dependency "ofi" "${DEP_CONFIG[@]}"; then - clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.8.1" & + clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.12.1" & fi fi @@ -404,11 +404,8 @@ if check_dependency "psm2" "${DEP_CONFIG[@]}"; then fi # get Mercury -if check_dependency "mercury-experimental" "${DEP_CONFIG[@]}"; then - # currently not compatible with Hermes +if check_dependency "mercury" "${DEP_CONFIG[@]}"; then clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "75cb615894699a4df5872ba9d187f37cc94330a3" "--recurse-submodules" & -elif check_dependency "mercury" "${DEP_CONFIG[@]}"; then - clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "4796aeeb90ba58be8a3b17a73f27aa3afed1ee0f" "--recurse-submodules" & fi # get Argobots @@ -422,10 +419,8 @@ if check_dependency "margo" "${DEP_CONFIG[@]}"; then fi # get rocksdb -if check_dependency "rocksdb-experimental" "${DEP_CONFIG[@]}"; then +if check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/refs/tags/v6.16.4.tar.gz" & -elif check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then - wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz" & fi # get syscall_intercept -- GitLab From 4c16cfb6196df726bb2cc9fd1e78ff01ac1130e5 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Tue, 27 Apr 2021 00:22:17 +0200 Subject: [PATCH 13/16] CI debug --- scripts/compile_dep.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 1acaa7774..d6e6cc94f 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -380,6 +380,13 @@ if check_dependency "json-c" "${DEP_CONFIG[@]}"; then cd ${CURR}/build ../cmake-configure --prefix=${INSTALL} make -j${CORES} install + echo "XXX1" + ls -l ${INSTALL} + echo "XXX2" + ls -l ${INSTALL}/lib64/ + echo "XXX3" + ls -l ${INSTALL}/lib64/pkgconfig + echo "XXX4" mv ${INSTALL}/lib64/pkgconfig/json-c.pc ${INSTALL}/lib/pkgconfig/ fi -- GitLab From 347a68676fbb852dca2a0fc26ee347f348b4e29f Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Tue, 27 Apr 2021 00:26:39 +0200 Subject: [PATCH 14/16] update hermes --- external/hermes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/hermes b/external/hermes index ca72cf9c3..97820ed81 160000 --- a/external/hermes +++ b/external/hermes @@ -1 +1 @@ -Subproject commit ca72cf9c33d5d7f84c689c00b5b50583d570cead +Subproject commit 97820ed81aba9ef7c05891ce80c53a8533b78609 -- GitLab From 1201e699a13c31b506ef075ffebcc5b45a9fb9d1 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Tue, 27 Apr 2021 00:27:34 +0200 Subject: [PATCH 15/16] CI fix --- scripts/compile_dep.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index d6e6cc94f..8721dee1d 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -377,16 +377,10 @@ if check_dependency "json-c" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: json-c" CURR=${SOURCE}/json-c prepare_build_dir ${CURR} + mkdir -p ${INSTALL}/lib64/pkgconfig cd ${CURR}/build ../cmake-configure --prefix=${INSTALL} make -j${CORES} install - echo "XXX1" - ls -l ${INSTALL} - echo "XXX2" - ls -l ${INSTALL}/lib64/ - echo "XXX3" - ls -l ${INSTALL}/lib64/pkgconfig - echo "XXX4" mv ${INSTALL}/lib64/pkgconfig/json-c.pc ${INSTALL}/lib/pkgconfig/ fi -- GitLab From 1d39292a90348ae0fe21142ad3596b66809c7d1f Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Tue, 27 Apr 2021 00:32:16 +0200 Subject: [PATCH 16/16] CI fix ... --- scripts/compile_dep.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 8721dee1d..5f1f3f189 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -377,11 +377,11 @@ if check_dependency "json-c" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: json-c" CURR=${SOURCE}/json-c prepare_build_dir ${CURR} - mkdir -p ${INSTALL}/lib64/pkgconfig cd ${CURR}/build ../cmake-configure --prefix=${INSTALL} make -j${CORES} install - mv ${INSTALL}/lib64/pkgconfig/json-c.pc ${INSTALL}/lib/pkgconfig/ + mkdir -p ${INSTALL}/lib/pkgconfig + find ${INSTALL} -type f -name "json-c.pc" -exec mv {} ${INSTALL}/lib/pkgconfig \; fi # build BMI -- GitLab