Commit efdce80e authored by Alberto Miranda's avatar Alberto Miranda ♨️
Browse files

Merge branch '98-add-ofi-verbs-support' into 'master' 

Resolve "Add ofi+verbs support"

Closes #98

See merge request !38
parents 9ceaca59 f6a49aa2

hermes @ a25a82d4

Compare 31d15c23 to a25a82d4
Original line number Original line Diff line number Diff line 
Subproject commit 31d15c2339d48dba6561546a45ecd759c16d8455
 
Subproject commit a25a82d401b4a0d986088c6802a08fd79185a278

include/global/global_defs.hpp

+1 −0
Original line number Original line Diff line number Diff line
@@ -42,6 +42,7 @@ namespace protocol { 
constexpr auto ofi_psm2 = "ofi+psm2";

constexpr auto ofi_psm2 = "ofi+psm2";

constexpr auto ofi_sockets = "ofi+sockets";

constexpr auto ofi_sockets = "ofi+sockets";

constexpr auto ofi_tcp = "ofi+tcp";

constexpr auto ofi_tcp = "ofi+tcp";

constexpr auto ofi_verbs = "ofi+verbs";

} // namespace protocol

} // namespace protocol

} // namespace rpc

} // namespace rpc

scripts/compile_dep.sh

+20 −4
Original line number Original line Diff line number Diff line
@@ -9,7 +9,12 @@ SOURCE="" 
INSTALL=""

INSTALL=""

DEP_CONFIG=""

DEP_CONFIG=""





VALID_DEP_OPTIONS="mogon2 direct all"

VALID_DEP_OPTIONS="mogon2 mogon1 direct all"



MOGON1_DEPS=(

    "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"

    "syscall_intercept" "date" "verbs"

)





MOGON2_DEPS=(

MOGON2_DEPS=(

    "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"

    "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"
@@ -52,7 +57,7 @@ optional arguments: 
                defaults to 'all'

                defaults to 'all'

    -c <CONFIG>, --config <CONFIG>

    -c <CONFIG>, --config <CONFIG>

                allows additional configurations, e.g., for specific clusters

                allows additional configurations, e.g., for specific clusters

                supported values: {mogon2, direct, all}

                supported values: {mogon1, mogon2, direct, all}

                defaults to 'direct'

                defaults to 'direct'

    -d <DEPENDENCY>, --dependency <DEPENDENCY>

    -d <DEPENDENCY>, --dependency <DEPENDENCY>

                download a specific dependency and ignore --config setting. If unspecified

                download a specific dependency and ignore --config setting. If unspecified
@@ -69,6 +74,10 @@ list_dependencies() { 




    echo "Available dependencies: "

    echo "Available dependencies: "





    echo -n "  Mogon 1: "

    for d in "${MOGON1_DEPS[@]}"; do

        echo -n "$d "

    done

    echo -n "  Mogon 2: "

    echo -n "  Mogon 2: "

    for d in "${MOGON2_DEPS[@]}"; do

    for d in "${MOGON2_DEPS[@]}"; do

        echo -n "$d "

        echo -n "$d "
@@ -94,7 +103,6 @@ check_dependency() { 
      if echo "${DEPENDENCY}" | grep -q "${DEP}"; then

      if echo "${DEPENDENCY}" | grep -q "${DEP}"; then

        return

        return

      fi

      fi

#      [[ "${DEPENDENCY}" == "${DEP}" ]] && return

  else

  else

      # if not check if dependency is part of dependency config

      # if not check if dependency is part of dependency config

      for e in "${DEP_CONFIG[@]}"; do

      for e in "${DEP_CONFIG[@]}"; do
@@ -217,6 +225,10 @@ else 
fi

fi

# enable predefined dependency template

# enable predefined dependency template

case ${TMP_DEP_CONF} in

case ${TMP_DEP_CONF} in

mogon1)

  DEP_CONFIG=("${MOGON1_DEPS[@]}")

  echo "'Mogon1' dependencies are compiled"

  ;;

mogon2)

mogon2)

  DEP_CONFIG=("${MOGON2_DEPS[@]}")

  DEP_CONFIG=("${MOGON2_DEPS[@]}")

  echo "'Mogon2' dependencies are compiled"

  echo "'Mogon2' dependencies are compiled"
@@ -320,7 +332,11 @@ if check_dependency "ofi" "${DEP_CONFIG[@]}"; then 
        CURR=${SOURCE}/libfabric

        CURR=${SOURCE}/libfabric

        prepare_build_dir ${CURR}

        prepare_build_dir ${CURR}

        cd ${CURR}/build

        cd ${CURR}/build

        ../configure --prefix=${INSTALL} --enable-tcp=yes

        OFI_CONFIG="../configure --prefix=${INSTALL} --enable-tcp=yes"

        if check_dependency "verbs" "${DEP_CONFIG[@]}"; then

            OFI_CONFIG="${OFI_CONFIG} --enable-verbs=yes"

        fi

         ${OFI_CONFIG}

        make -j${CORES}

        make -j${CORES}

        make install

        make install

        [ "${PERFORM_TEST}" ] && make check

        [ "${PERFORM_TEST}" ] && make check

scripts/dl_dep.sh

+25 −4
Original line number Original line Diff line number Diff line
@@ -9,7 +9,12 @@ NA_LAYER="" 
DEP_CONFIG=""

DEP_CONFIG=""

VERBOSE=false

VERBOSE=false





VALID_DEP_OPTIONS="mogon2 direct all"

VALID_DEP_OPTIONS="mogon2 mogon1 direct all"



MOGON1_DEPS=(

    "zstd" "lz4" "snappy" "capstone" "ofi-verbs" "mercury" "argobots" "margo" "rocksdb"

    "syscall_intercept" "date"

)





MOGON2_DEPS=(

MOGON2_DEPS=(

    "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"

    "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"
@@ -48,6 +53,10 @@ list_dependencies() { 




    echo "Available dependencies: "

    echo "Available dependencies: "





    echo -n "  Mogon 1: "

    for d in "${MOGON1_DEPS[@]}"; do

        echo -n "$d "

    done

    echo -n "  Mogon 2: "

    echo -n "  Mogon 2: "

    for d in "${MOGON2_DEPS[@]}"; do

    for d in "${MOGON2_DEPS[@]}"; do

        echo -n "$d "

        echo -n "$d "
@@ -248,6 +257,10 @@ fi 




# enable predefined dependency template

# enable predefined dependency template

case ${TMP_DEP_CONF} in

case ${TMP_DEP_CONF} in

mogon1)

  DEP_CONFIG=("${MOGON1_DEPS[@]}")

  [[ -z "${DEPENDENCY}" ]] && echo "'Mogon1' dependencies are downloaded"

  ;;

mogon2)

mogon2)

  DEP_CONFIG=("${MOGON2_DEPS[@]}")

  DEP_CONFIG=("${MOGON2_DEPS[@]}")

  [[ -z "${DEPENDENCY}" ]] && echo "'Mogon2' dependencies are downloaded"

  [[ -z "${DEPENDENCY}" ]] && echo "'Mogon2' dependencies are downloaded"
@@ -308,8 +321,14 @@ if check_dependency "bmi" "${DEP_CONFIG[@]}"; then 
fi

fi





# get libfabric

# get libfabric

if check_dependency "ofi" "${DEP_CONFIG[@]}"; then

if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then

if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then

    if check_dependency "ofi-experimental" "${DEP_CONFIG[@]}"; then

        wgetdeps "libfabric" "https://github.com/ofiwg/libfabric/releases/download/v1.9.1/libfabric-1.9.1.tar.bz2" &

    elif check_dependency "ofi-verbs" "${DEP_CONFIG[@]}"; then

        # libibverbs 1.2.1-1 used on mogon 1i (installed on system) which is linked to libfabric

        # libfabric 1.8 random RPCs fail to be send. 1.9 RPC client cannot be started when in an MPI environment

        wgetdeps "libfabric" "https://github.com/ofiwg/libfabric/releases/download/v1.7.2/libfabric-1.7.2.tar.gz" &

    elif check_dependency "ofi" "${DEP_CONFIG[@]}"; then

        wgetdeps "libfabric" "https://github.com/ofiwg/libfabric/releases/download/v1.8.1/libfabric-1.8.1.tar.bz2" &

        wgetdeps "libfabric" "https://github.com/ofiwg/libfabric/releases/download/v1.8.1/libfabric-1.8.1.tar.bz2" &

    fi

    fi

fi

fi
@@ -332,6 +351,8 @@ fi 
# get rocksdb

# get rocksdb

if check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then

if check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then

    wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz" &

    wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz" &

elif check_dependency "rocksdb-experimental" "${DEP_CONFIG[@]}"; then

    wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.7.3.tar.gz" &

fi

fi





# get syscall_intercept

# get syscall_intercept

src/daemon/daemon.cpp

+24 −7
Original line number Original line Diff line number Diff line
@@ -32,9 +32,13 @@ 
#include <iostream>

#include <iostream>

#include <fstream>

#include <fstream>

#include <csignal>

#include <csignal>

#include <unistd.h>

#include <condition_variable>

#include <condition_variable>





extern "C" {

#include <unistd.h>

#include <stdlib.h>

}



using namespace std;

using namespace std;

namespace po = boost::program_options;

namespace po = boost::program_options;

namespace bfs = boost::filesystem;

namespace bfs = boost::filesystem;
@@ -279,11 +283,14 @@ int main(int argc, const char* argv[]) { 
            ("mountdir,m", po::value<string>()->required(), "User Fuse mountdir")

            ("mountdir,m", po::value<string>()->required(), "User Fuse mountdir")

            ("rootdir,r", po::value<string>()->required(), "data directory")

            ("rootdir,r", po::value<string>()->required(), "data directory")

            ("metadir,i", po::value<string>(), "metadata directory, if not set rootdir is used for metadata ")

            ("metadir,i", po::value<string>(), "metadata directory, if not set rootdir is used for metadata ")

            ("listen,l", po::value<string>(), "Address or interface to bind the daemon on. Default: local hostname")

            ("listen,l", po::value<string>(), "Address or interface to bind the daemon on. Default: local hostname.\n"

                                              "When used with ofi+verbs the FI_VERBS_IFACE environment variable is set accordingly "

                                              "which associates the verbs device with the network interface. In case FI_VERBS_IFACE "

                                              "is already defined, the argument is ignored. Default 'ib'.")

            ("hosts-file,H", po::value<string>(),

            ("hosts-file,H", po::value<string>(),

             "Shared file used by deamons to register their "

             "Shared file used by deamons to register their "

             "enpoints. (default './gkfs_hosts.txt')")

             "enpoints. (default './gkfs_hosts.txt')")

            ("version,h", "print version and exit");

            ("version", "print version and exit");

    po::variables_map vm;

    po::variables_map vm;

    po::store(po::parse_command_line(argc, argv, desc), vm);

    po::store(po::parse_command_line(argc, argv, desc), vm);
@@ -324,11 +331,21 @@ int main(int argc, const char* argv[]) { 
    initialize_loggers();

    initialize_loggers();

    GKFS_DATA->spdlogger(spdlog::get("main"));

    GKFS_DATA->spdlogger(spdlog::get("main"));







    string addr{};

    string addr;

    if (vm.count("listen")) {

    if (vm.count("listen")) {

        addr = vm["listen"].as<string>();

        addr = vm["listen"].as<string>();

        // ofi+verbs requires an empty addr to bind to the ib interface

        if (RPC_PROTOCOL == string(gkfs::rpc::protocol::ofi_verbs)) {

            /*

             * FI_VERBS_IFACE : The prefix or the full name of the network interface associated with the verbs device (default: ib)

             * Mercury does not allow to bind to an address when ofi+verbs is used

             */

            if (!secure_getenv("FI_VERBS_IFACE"))

                setenv("FI_VERBS_IFACE", addr.c_str(), 1);

            addr = ""s;

        }

    } else {

    } else {

        if (RPC_PROTOCOL != string(gkfs::rpc::protocol::ofi_verbs))

            addr = get_my_hostname(true);

            addr = get_my_hostname(true);

    }

    }