Merge branch '59-native-omni-path-support-2' into 'master' (2a236e33) · Commits · hpc / gekkofs

hermes @ 0af45bfa

Original line number	Diff line number	Diff line
		Subproject commit a25a82d401b4a0d986088c6802a08fd79185a278
		Subproject commit 0af45bfa667f7ff9c78167ef94d975bffbd879f0

scripts/compile_dep.sh

+26 −4

Original line number	Diff line number	Diff line
		@@ -9,7 +9,7 @@ SOURCE=""
		INSTALL=""
		DEP_CONFIG=""

		VALID_DEP_OPTIONS="mogon2 mogon1 direct all"
		VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all"

		MOGON1_DEPS=(
		"zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"
		@@ -18,7 +18,12 @@ MOGON1_DEPS=(

		MOGON2_DEPS=(
		"zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"
		"syscall_intercept" "date"
		"syscall_intercept" "date" "psm2"
		)

		NGIO_DEPS=(
		"zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"
		"syscall_intercept" "date" "psm2"
		)

		DIRECT_DEPS=(
		@@ -57,7 +62,7 @@ optional arguments:
		defaults to 'all'
		-c <CONFIG>, --config <CONFIG>
		allows additional configurations, e.g., for specific clusters
		supported values: {mogon1, mogon2, direct, all}
		supported values: {mogon1, mogon2, ngio, direct, all}
		defaults to 'direct'
		-d <DEPENDENCY>, --dependency <DEPENDENCY>
		download a specific dependency and ignore --config setting. If unspecified
		@@ -78,19 +83,27 @@ list_dependencies() {
		for d in "${MOGON1_DEPS[@]}"; do
		echo -n "$d "
		done
		echo
		echo -n " Mogon 2: "
		for d in "${MOGON2_DEPS[@]}"; do
		echo -n "$d "
		done
		echo
		echo -n " NGIO: "
		for d in "${NGIO_DEPS[@]}"; do
		echo -n "$d "
		done
		echo
		echo -n " Direct GekkoFS dependencies: "
		for d in "${DIRECT_DEPS[@]}"; do
		echo -n "$d "
		done
		echo
		echo -n " All: "
		for d in "${ALL_DEPS[@]}"; do
		echo -n "$d "
		done
		echo ""
		echo
		}

		check_dependency() {
		@@ -233,6 +246,10 @@ mogon2)
		DEP_CONFIG=("${MOGON2_DEPS[@]}")
		echo "'Mogon2' dependencies are compiled"
		;;
		ngio)
		DEP_CONFIG=("${NGIO_DEPS[@]}")
		echo "'NGIO' dependencies are compiled"
		;;
		all)
		DEP_CONFIG=("${ALL_DEPS[@]}")
		echo "'All' dependencies are compiled"
		@@ -260,6 +277,7 @@ set -e

		export CPATH="${CPATH}:${INSTALL}/include"
		export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL}/lib:${INSTALL}/lib64"
		export PKG_CONFIG_PATH="${INSTALL}/lib/pkgconfig:${PKG_CONFIG_PATH}"

		## Third party dependencies

		@@ -337,6 +355,10 @@ if check_dependency "ofi" "${DEP_CONFIG[@]}"; then
		OFI_CONFIG="../configure --prefix=${INSTALL} --enable-tcp=yes"
		if check_dependency "verbs" "${DEP_CONFIG[@]}"; then
		OFI_CONFIG="${OFI_CONFIG} --enable-verbs=yes"
		elif check_dependency "psm2" "${DEP_CONFIG[@]}"; then
		OFI_CONFIG="${OFI_CONFIG} --enable-psm2=yes --with-psm2-src=${SOURCE}/psm2"
		elif check_dependency "psm2-system" "${DEP_CONFIG[@]}"; then
		OFI_CONFIG="${OFI_CONFIG} --enable-psm2=yes"
		fi
		${OFI_CONFIG}
		make -j${CORES}

scripts/dl_dep.sh

+26 −6

Original line number	Diff line number	Diff line
		@@ -9,7 +9,7 @@ NA_LAYER=""
		DEP_CONFIG=""
		VERBOSE=false

		VALID_DEP_OPTIONS="mogon2 mogon1 direct all"
		VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all"

		MOGON1_DEPS=(
		"zstd" "lz4" "snappy" "capstone" "ofi-verbs" "mercury" "argobots" "margo" "rocksdb"
		@@ -21,6 +21,11 @@ MOGON2_DEPS=(
		"syscall_intercept-glibc3" "date" "psm2"
		)

		NGIO_DEPS=(
		"zstd" "lz4" "snappy" "capstone" "ofi-experimental" "mercury" "argobots" "margo" "rocksdb"
		"syscall_intercept" "date" "psm2"

		)
		DIRECT_DEPS=(
		"ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date"
		)
		@@ -57,19 +62,27 @@ list_dependencies() {
		for d in "${MOGON1_DEPS[@]}"; do
		echo -n "$d "
		done
		echo
		echo -n " Mogon 2: "
		for d in "${MOGON2_DEPS[@]}"; do
		echo -n "$d "
		done
		echo
		echo -n " NGIO: "
		for d in "${NGIO_DEPS[@]}"; do
		echo -n "$d "
		done
		echo
		echo -n " Direct GekkoFS dependencies: "
		for d in "${DIRECT_DEPS[@]}"; do
		echo -n "$d "
		done
		echo
		echo -n " All: "
		for d in "${ALL_DEPS[@]}"; do
		echo -n "$d "
		done
		echo ""
		echo
		}

		check_dependency() {
		@@ -82,7 +95,6 @@ check_dependency() {
		if echo "${DEPENDENCY}" \| grep -q "${DEP}"; then
		return
		fi
		# [[ "${DEPENDENCY}" == "${DEP}" ]] && return
		else
		# if not check if dependency is part of dependency config
		for e in "${DEP_CONFIG[@]}"; do
		@@ -179,7 +191,7 @@ optional arguments:
		defaults to 'ofi'
		-c <CONFIG>, --config <CONFIG>
		allows additional configurations, e.g., for specific clusters
		supported values: {mogon2, direct, all}
		supported values: {mogon2, mogon1, ngio, direct, all}
		defaults to 'direct'
		-d <DEPENDENCY>, --dependency <DEPENDENCY>
		download a specific dependency and ignore --config setting. If unspecified
		@@ -265,6 +277,10 @@ mogon2)
		DEP_CONFIG=("${MOGON2_DEPS[@]}")
		[[ -z "${DEPENDENCY}" ]] && echo "'Mogon2' dependencies are downloaded"
		;;
		ngio)
		DEP_CONFIG=("${NGIO_DEPS[@]}")
		[[ -z "${DEPENDENCY}" ]] && echo "'NGIO' dependencies are downloaded"
		;;
		all)
		DEP_CONFIG=("${ALL_DEPS[@]}")
		[[ -z "${DEPENDENCY}" ]] && echo "'All' dependencies are downloaded"
		@@ -333,9 +349,13 @@ if [ "${NA_LAYER}" == "ofi" ] \|\| [ "${NA_LAYER}" == "all" ]; then
		fi
		fi

		if check_dependency "psm2" "${DEP_CONFIG[@]}"; then
		wgetdeps "psm2" "https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz" &
		fi

		# get Mercury
		if check_dependency "mercury" "${DEP_CONFIG[@]}"; then
		clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "fd410dfb9852b2b98d21113531f3058f45bfcd64" "--recurse-submodules" &
		clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "41caa143a07ed179a3149cac4af0dc7aa3f946fd" "--recurse-submodules" &
		fi

		# get Argobots
		@@ -345,7 +365,7 @@ fi

		# get Margo
		if check_dependency "margo" "${DEP_CONFIG[@]}"; then
		clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "016dbdce22da3fe4f97b46c20a53bced9370a217" &
		clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "v0.6.3" &
		fi

		# get rocksdb

src/client/preload.cpp

+4 −0

Original line number	Diff line number	Diff line
		@@ -19,6 +19,7 @@
		#include <client/intercept.hpp>

		#include <global/rpc/distributor.hpp>
		#include <global/global_defs.hpp>

		#include <fstream>

		@@ -64,6 +65,9 @@ bool init_hermes_client(const std::string& transport_prefix) {
		#if USE_SHM
		opts \|= hermes::use_auto_sm;
		#endif
		if (gkfs::rpc::protocol::ofi_psm2 == string(RPC_PROTOCOL)) {
		opts \|= hermes::force_no_block_progress;
		}

		ld_network_service =
		std::make_unique<hermes::async_engine>(

src/daemon/daemon.cpp

+3 −2

Original line number	Diff line number	Diff line
		@@ -103,8 +103,7 @@ void init_rpc_server(const string& protocol_port) {
		hg_addr_t addr_self;
		hg_size_t addr_self_cstring_sz = 128;
		char addr_self_cstring[128];
		// IMPORTANT: this struct needs to be zeroed before use
		struct hg_init_info hg_options = {};
		struct hg_init_info hg_options = HG_INIT_INFO_INITIALIZER;
		#if USE_SHM
		hg_options.auto_sm = HG_TRUE;
		#else
		@@ -112,6 +111,8 @@ void init_rpc_server(const string& protocol_port) {
		#endif
		hg_options.stats = HG_FALSE;
		hg_options.na_class = nullptr;
		if (gkfs::rpc::protocol::ofi_psm2 == string(RPC_PROTOCOL))
		hg_options.na_init_info.progress_mode = NA_NO_BLOCK;
		// Start Margo (this will also initialize Argobots and Mercury internally)
		auto mid = margo_init_opt(protocol_port.c_str(),
		MARGO_SERVER_MODE,