Resolve "Native Omni-Path support"

Marc Vefrequested to merge
59-native-omni-path-support-2 into master

Works, when it works... Tested on Mogon2 and NGIO. Includes a new dependency set for NGIO.

Incompatible with IO500 because disconnects are not done properly.

Depends on Pull Request on Hermes: https://github.com/bsc-ssrg/hermes/pull/9 and is ready to merge after.

Closes #59 (closed)

Edited by Marc Vef

Merge request reports

hermes @ 0af45bfa

Compare a25a82d4 to 0af45bfa
Changes for external/hermes: 1 added line, 1 removed line.
Original line number Diff line number Diff line 
Subproject commit a25a82d401b4a0d986088c6802a08fd79185a278
 
Subproject commit 0af45bfa667f7ff9c78167ef94d975bffbd879f0

scripts/compile_dep.sh

+26 −4
Changes for scripts/compile_dep.sh: 26 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -9,7 +9,7 @@ SOURCE="" 
INSTALL=""

DEP_CONFIG=""



VALID_DEP_OPTIONS="mogon2 mogon1 direct all"

VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all"



MOGON1_DEPS=(

    "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"
@@ -18,7 +18,12 @@ MOGON1_DEPS=( 


MOGON2_DEPS=(

    "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"

    "syscall_intercept" "date"

    "syscall_intercept" "date" "psm2"

)



NGIO_DEPS=(

    "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"

    "syscall_intercept" "date" "psm2"

)



DIRECT_DEPS=(
@@ -57,7 +62,7 @@ optional arguments: 
                defaults to 'all'

    -c <CONFIG>, --config <CONFIG>

                allows additional configurations, e.g., for specific clusters

                supported values: {mogon1, mogon2, direct, all}

                supported values: {mogon1, mogon2, ngio, direct, all}

                defaults to 'direct'

    -d <DEPENDENCY>, --dependency <DEPENDENCY>

                download a specific dependency and ignore --config setting. If unspecified
@@ -78,19 +83,27 @@ list_dependencies() { 
    for d in "${MOGON1_DEPS[@]}"; do

        echo -n "$d "

    done

	echo

    echo -n "  Mogon 2: "

    for d in "${MOGON2_DEPS[@]}"; do

        echo -n "$d "

    done

	echo

    echo -n "  NGIO: "

    for d in "${NGIO_DEPS[@]}"; do

        echo -n "$d "

    done

	echo

    echo -n "  Direct GekkoFS dependencies: "

    for d in "${DIRECT_DEPS[@]}"; do

        echo -n "$d "

    done

	echo

    echo -n "  All: "

    for d in "${ALL_DEPS[@]}"; do

        echo -n "$d "

    done

    echo ""

    echo

}



check_dependency() {
@@ -233,6 +246,10 @@ mogon2) 
  DEP_CONFIG=("${MOGON2_DEPS[@]}")

  echo "'Mogon2' dependencies are compiled"

  ;;

ngio)

  DEP_CONFIG=("${NGIO_DEPS[@]}")

  echo "'NGIO' dependencies are compiled"

  ;;

all)

  DEP_CONFIG=("${ALL_DEPS[@]}")

  echo "'All' dependencies are compiled"
@@ -260,6 +277,7 @@ set -e 


export CPATH="${CPATH}:${INSTALL}/include"

export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL}/lib:${INSTALL}/lib64"

export PKG_CONFIG_PATH="${INSTALL}/lib/pkgconfig:${PKG_CONFIG_PATH}"



## Third party dependencies
@@ -337,6 +355,10 @@ if check_dependency "ofi" "${DEP_CONFIG[@]}"; then 
        OFI_CONFIG="../configure --prefix=${INSTALL} --enable-tcp=yes"

        if check_dependency "verbs" "${DEP_CONFIG[@]}"; then

            OFI_CONFIG="${OFI_CONFIG} --enable-verbs=yes"

        elif check_dependency "psm2" "${DEP_CONFIG[@]}"; then

            OFI_CONFIG="${OFI_CONFIG} --enable-psm2=yes --with-psm2-src=${SOURCE}/psm2"

        elif check_dependency "psm2-system" "${DEP_CONFIG[@]}"; then

            OFI_CONFIG="${OFI_CONFIG} --enable-psm2=yes"

        fi

         ${OFI_CONFIG}

        make -j${CORES}

scripts/dl_dep.sh

+26 −6
Changes for scripts/dl_dep.sh: 26 added lines, 6 removed lines.
Original line number Diff line number Diff line
@@ -9,7 +9,7 @@ NA_LAYER="" 
DEP_CONFIG=""

VERBOSE=false



VALID_DEP_OPTIONS="mogon2 mogon1 direct all"

VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all"



MOGON1_DEPS=(

    "zstd" "lz4" "snappy" "capstone" "ofi-verbs" "mercury" "argobots" "margo" "rocksdb"
@@ -21,6 +21,11 @@ MOGON2_DEPS=( 
    "syscall_intercept-glibc3" "date" "psm2"

)



NGIO_DEPS=(

    "zstd" "lz4" "snappy" "capstone" "ofi-experimental" "mercury" "argobots" "margo" "rocksdb"

    "syscall_intercept" "date" "psm2"



)

DIRECT_DEPS=(

  "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date"

)
@@ -57,19 +62,27 @@ list_dependencies() { 
    for d in "${MOGON1_DEPS[@]}"; do

        echo -n "$d "

    done

	echo

    echo -n "  Mogon 2: "

    for d in "${MOGON2_DEPS[@]}"; do

        echo -n "$d "

    done

	echo

    echo -n "  NGIO: "

    for d in "${NGIO_DEPS[@]}"; do

        echo -n "$d "

    done

	echo

    echo -n "  Direct GekkoFS dependencies: "

    for d in "${DIRECT_DEPS[@]}"; do

        echo -n "$d "

    done

	echo

    echo -n "  All: "

    for d in "${ALL_DEPS[@]}"; do

        echo -n "$d "

    done

    echo ""

    echo 

}



check_dependency() {
@@ -82,7 +95,6 @@ check_dependency() { 
      if echo "${DEPENDENCY}" | grep -q "${DEP}"; then

        return

      fi

#      [[ "${DEPENDENCY}" == "${DEP}" ]] && return

  else

      # if not check if dependency is part of dependency config

      for e in "${DEP_CONFIG[@]}"; do
@@ -179,7 +191,7 @@ optional arguments: 
                                defaults to 'ofi'

        -c <CONFIG>, --config <CONFIG>

                                allows additional configurations, e.g., for specific clusters

                                supported values: {mogon2, direct, all}

                                supported values: {mogon2, mogon1, ngio, direct, all}

                                defaults to 'direct'

        -d <DEPENDENCY>, --dependency <DEPENDENCY>

                                download a specific dependency and ignore --config setting. If unspecified
@@ -265,6 +277,10 @@ mogon2) 
  DEP_CONFIG=("${MOGON2_DEPS[@]}")

  [[ -z "${DEPENDENCY}" ]] && echo "'Mogon2' dependencies are downloaded"

  ;;

ngio)

  DEP_CONFIG=("${NGIO_DEPS[@]}")

  [[ -z "${DEPENDENCY}" ]] && echo "'NGIO' dependencies are downloaded"

  ;;

all)

  DEP_CONFIG=("${ALL_DEPS[@]}")

  [[ -z "${DEPENDENCY}" ]] && echo "'All' dependencies are downloaded"
@@ -333,9 +349,13 @@ if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then 
    fi

fi



if check_dependency "psm2" "${DEP_CONFIG[@]}"; then

    wgetdeps "psm2" "https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz" &

fi



# get Mercury

if check_dependency "mercury" "${DEP_CONFIG[@]}"; then

    clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "fd410dfb9852b2b98d21113531f3058f45bfcd64"  "--recurse-submodules" &

    clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "41caa143a07ed179a3149cac4af0dc7aa3f946fd" "--recurse-submodules" &

fi



# get Argobots
@@ -345,7 +365,7 @@ fi 


# get Margo

if check_dependency "margo" "${DEP_CONFIG[@]}"; then

    clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "016dbdce22da3fe4f97b46c20a53bced9370a217" &

    clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "v0.6.3" &

fi



# get rocksdb

src/client/preload.cpp

+4 −0
Changes for src/client/preload.cpp: 4 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include <client/intercept.hpp>



#include <global/rpc/distributor.hpp>

#include <global/global_defs.hpp>



#include <fstream>
@@ -64,6 +65,9 @@ bool init_hermes_client(const std::string& transport_prefix) { 
#if USE_SHM

        opts |= hermes::use_auto_sm;

#endif

        if (gkfs::rpc::protocol::ofi_psm2 == string(RPC_PROTOCOL)) {

            opts |= hermes::force_no_block_progress;

        }



        ld_network_service =

                std::make_unique<hermes::async_engine>(

src/daemon/daemon.cpp

+3 −2
Changes for src/daemon/daemon.cpp: 3 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -103,8 +103,7 @@ void init_rpc_server(const string& protocol_port) { 
    hg_addr_t addr_self;

    hg_size_t addr_self_cstring_sz = 128;

    char addr_self_cstring[128];

    // IMPORTANT: this struct needs to be zeroed before use

    struct hg_init_info hg_options = {};

    struct hg_init_info hg_options = HG_INIT_INFO_INITIALIZER;

#if USE_SHM

    hg_options.auto_sm = HG_TRUE;

#else
@@ -112,6 +111,8 @@ void init_rpc_server(const string& protocol_port) { 
#endif

    hg_options.stats = HG_FALSE;

    hg_options.na_class = nullptr;

    if (gkfs::rpc::protocol::ofi_psm2 == string(RPC_PROTOCOL))

        hg_options.na_init_info.progress_mode = NA_NO_BLOCK;

    // Start Margo (this will also initialize Argobots and Mercury internally)

    auto mid = margo_init_opt(protocol_port.c_str(),

                              MARGO_SERVER_MODE,