diff --git a/external/hermes b/external/hermes index a25a82d401b4a0d986088c6802a08fd79185a278..0af45bfa667f7ff9c78167ef94d975bffbd879f0 160000 --- a/external/hermes +++ b/external/hermes @@ -1 +1 @@ -Subproject commit a25a82d401b4a0d986088c6802a08fd79185a278 +Subproject commit 0af45bfa667f7ff9c78167ef94d975bffbd879f0 diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 51d7e94b4484c583904befca5b69ecb4c2d91a8d..79a0ad69249b46e149067b1e270eae5db313781e 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -9,7 +9,7 @@ SOURCE="" INSTALL="" DEP_CONFIG="" -VALID_DEP_OPTIONS="mogon2 mogon1 direct all" +VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all" MOGON1_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" @@ -18,7 +18,12 @@ MOGON1_DEPS=( MOGON2_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "syscall_intercept" "date" "psm2" +) + +NGIO_DEPS=( + "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" "date" "psm2" ) DIRECT_DEPS=( @@ -57,7 +62,7 @@ optional arguments: defaults to 'all' -c , --config allows additional configurations, e.g., for specific clusters - supported values: {mogon1, mogon2, direct, all} + supported values: {mogon1, mogon2, ngio, direct, all} defaults to 'direct' -d , --dependency download a specific dependency and ignore --config setting. If unspecified @@ -78,19 +83,27 @@ list_dependencies() { for d in "${MOGON1_DEPS[@]}"; do echo -n "$d " done + echo echo -n " Mogon 2: " for d in "${MOGON2_DEPS[@]}"; do echo -n "$d " done + echo + echo -n " NGIO: " + for d in "${NGIO_DEPS[@]}"; do + echo -n "$d " + done + echo echo -n " Direct GekkoFS dependencies: " for d in "${DIRECT_DEPS[@]}"; do echo -n "$d " done + echo echo -n " All: " for d in "${ALL_DEPS[@]}"; do echo -n "$d " done - echo "" + echo } check_dependency() { @@ -233,6 +246,10 @@ mogon2) DEP_CONFIG=("${MOGON2_DEPS[@]}") echo "'Mogon2' dependencies are compiled" ;; +ngio) + DEP_CONFIG=("${NGIO_DEPS[@]}") + echo "'NGIO' dependencies are compiled" + ;; all) DEP_CONFIG=("${ALL_DEPS[@]}") echo "'All' dependencies are compiled" @@ -260,6 +277,7 @@ set -e export CPATH="${CPATH}:${INSTALL}/include" export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL}/lib:${INSTALL}/lib64" +export PKG_CONFIG_PATH="${INSTALL}/lib/pkgconfig:${PKG_CONFIG_PATH}" ## Third party dependencies @@ -337,6 +355,10 @@ if check_dependency "ofi" "${DEP_CONFIG[@]}"; then OFI_CONFIG="../configure --prefix=${INSTALL} --enable-tcp=yes" if check_dependency "verbs" "${DEP_CONFIG[@]}"; then OFI_CONFIG="${OFI_CONFIG} --enable-verbs=yes" + elif check_dependency "psm2" "${DEP_CONFIG[@]}"; then + OFI_CONFIG="${OFI_CONFIG} --enable-psm2=yes --with-psm2-src=${SOURCE}/psm2" + elif check_dependency "psm2-system" "${DEP_CONFIG[@]}"; then + OFI_CONFIG="${OFI_CONFIG} --enable-psm2=yes" fi ${OFI_CONFIG} make -j${CORES} diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 60bd2b456f0c5ff4f1a346a3320c4e53e75c0aeb..d8cbff6a78daa4570abee50362a6a23fdbff639c 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -9,7 +9,7 @@ NA_LAYER="" DEP_CONFIG="" VERBOSE=false -VALID_DEP_OPTIONS="mogon2 mogon1 direct all" +VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all" MOGON1_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi-verbs" "mercury" "argobots" "margo" "rocksdb" @@ -21,6 +21,11 @@ MOGON2_DEPS=( "syscall_intercept-glibc3" "date" "psm2" ) +NGIO_DEPS=( + "zstd" "lz4" "snappy" "capstone" "ofi-experimental" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" "date" "psm2" + +) DIRECT_DEPS=( "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" ) @@ -57,19 +62,27 @@ list_dependencies() { for d in "${MOGON1_DEPS[@]}"; do echo -n "$d " done + echo echo -n " Mogon 2: " for d in "${MOGON2_DEPS[@]}"; do echo -n "$d " done + echo + echo -n " NGIO: " + for d in "${NGIO_DEPS[@]}"; do + echo -n "$d " + done + echo echo -n " Direct GekkoFS dependencies: " for d in "${DIRECT_DEPS[@]}"; do echo -n "$d " done + echo echo -n " All: " for d in "${ALL_DEPS[@]}"; do echo -n "$d " done - echo "" + echo } check_dependency() { @@ -82,7 +95,6 @@ check_dependency() { if echo "${DEPENDENCY}" | grep -q "${DEP}"; then return fi -# [[ "${DEPENDENCY}" == "${DEP}" ]] && return else # if not check if dependency is part of dependency config for e in "${DEP_CONFIG[@]}"; do @@ -179,7 +191,7 @@ optional arguments: defaults to 'ofi' -c , --config allows additional configurations, e.g., for specific clusters - supported values: {mogon2, direct, all} + supported values: {mogon2, mogon1, ngio, direct, all} defaults to 'direct' -d , --dependency download a specific dependency and ignore --config setting. If unspecified @@ -265,6 +277,10 @@ mogon2) DEP_CONFIG=("${MOGON2_DEPS[@]}") [[ -z "${DEPENDENCY}" ]] && echo "'Mogon2' dependencies are downloaded" ;; +ngio) + DEP_CONFIG=("${NGIO_DEPS[@]}") + [[ -z "${DEPENDENCY}" ]] && echo "'NGIO' dependencies are downloaded" + ;; all) DEP_CONFIG=("${ALL_DEPS[@]}") [[ -z "${DEPENDENCY}" ]] && echo "'All' dependencies are downloaded" @@ -333,9 +349,13 @@ if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then fi fi +if check_dependency "psm2" "${DEP_CONFIG[@]}"; then + wgetdeps "psm2" "https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz" & +fi + # get Mercury if check_dependency "mercury" "${DEP_CONFIG[@]}"; then - clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "fd410dfb9852b2b98d21113531f3058f45bfcd64" "--recurse-submodules" & + clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "41caa143a07ed179a3149cac4af0dc7aa3f946fd" "--recurse-submodules" & fi # get Argobots @@ -345,7 +365,7 @@ fi # get Margo if check_dependency "margo" "${DEP_CONFIG[@]}"; then - clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "016dbdce22da3fe4f97b46c20a53bced9370a217" & + clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "v0.6.3" & fi # get rocksdb diff --git a/src/client/preload.cpp b/src/client/preload.cpp index ad09cff3291be0ce05df43a4135e746eba5be6d9..a48c36c25c9c6902de56d6419c942ec48b2bbf77 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -19,6 +19,7 @@ #include #include +#include #include @@ -64,6 +65,9 @@ bool init_hermes_client(const std::string& transport_prefix) { #if USE_SHM opts |= hermes::use_auto_sm; #endif + if (gkfs::rpc::protocol::ofi_psm2 == string(RPC_PROTOCOL)) { + opts |= hermes::force_no_block_progress; + } ld_network_service = std::make_unique( diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 5a709f0e385a6b3da6bd1c04dcff8b758e70e4e2..a7a920627495254d71fd8cd063ac67208f5f925d 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -103,8 +103,7 @@ void init_rpc_server(const string& protocol_port) { hg_addr_t addr_self; hg_size_t addr_self_cstring_sz = 128; char addr_self_cstring[128]; - // IMPORTANT: this struct needs to be zeroed before use - struct hg_init_info hg_options = {}; + struct hg_init_info hg_options = HG_INIT_INFO_INITIALIZER; #if USE_SHM hg_options.auto_sm = HG_TRUE; #else @@ -112,6 +111,8 @@ void init_rpc_server(const string& protocol_port) { #endif hg_options.stats = HG_FALSE; hg_options.na_class = nullptr; + if (gkfs::rpc::protocol::ofi_psm2 == string(RPC_PROTOCOL)) + hg_options.na_init_info.progress_mode = NA_NO_BLOCK; // Start Margo (this will also initialize Argobots and Mercury internally) auto mid = margo_init_opt(protocol_port.c_str(), MARGO_SERVER_MODE,