Merge branch '45-mogon-ii-fix-dependency-compilation-scripts' into 'master' (1ec0d0fd) · Commits · hpc / gekkofs

scripts/compile_dep.sh

+243 −208

Original line number	Diff line number	Diff line
		#!/bin/bash

		MOGON1_DEPS=(
		"zstd" "lz4" "snappy" "bmi" "libfabric" "mercury" "argobots" "margo"
		"rocksdb" "syscall_intercept date"
		)
		PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
		PATCH_DIR="${PATCH_DIR}/patches"
		DEPENDENCY=""
		NA_LAYER=""
		CORES=""
		SOURCE=""
		INSTALL=""
		DEP_CONFIG=""

		VALID_DEP_OPTIONS="mogon2 direct all"

		MOGON2_DEPS=(
		"zstd" "lz4" "snappy" "bmi" "mercury" "argobots" "margo" "rocksdb"
		"syscall_intercept date"
		"zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"
		"syscall_intercept" "date"
		)

		DIRECT_DEPS=(
		"ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date"
		)

		FH2_DEPS=(
		"zstd" "lz4" "snappy" "bmi" "libfabric" "mercury" "argobots" "margo"
		"rocksdb" "syscall_intercept date"
		ALL_DEPS=(
		"zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb"
		"syscall_intercept" "date"
		)

		usage_short() {
		echo "
		usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CLUSTER>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>]
		usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CONFIG>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>]
		source_path install_path
		"
		}
		@@ -40,12 +50,14 @@ optional arguments:
		-n <NAPLUGIN>, --na <NAPLUGIN>
		network layer that is used for communication. Valid: {bmi,ofi,all}
		defaults to 'all'
		-c <CLUSTER>, --cluster <CLUSTER>
		additional configurations for specific compute clusters
		supported clusters: {mogon1,mogon2,fh2}
		-c <CONFIG>, --config <CONFIG>
		allows additional configurations, e.g., for specific clusters
		supported values: {mogon2, direct, all}
		defaults to 'direct'
		-d <DEPENDENCY>, --dependency <DEPENDENCY>
		build and install a specific dependency. If unspecified
		all dependencies are built and installed.
		download a specific dependency and ignore --config setting. If unspecified
		all dependencies are built and installed based on set --config setting.
		Multiple dependencies are supported: Pass a space-separated string (e.g., \"ofi mercury\"
		-j <COMPILE_CORES>, --compilecores <COMPILE_CORES>
		number of cores that are used to compile the dependencies
		defaults to number of available cores
		@@ -53,61 +65,66 @@ optional arguments:
		"
		}


		list_dependencies() {

		echo "Available dependencies: "

		echo -n " Mogon 1: "
		for d in "${MOGON1_DEPS[@]}"
		do
		echo -n " Mogon 2: "
		for d in "${MOGON2_DEPS[@]}"; do
		echo -n "$d "
		done
		echo ""

		echo -n " Mogon 2: "
		for d in "${MOGON2_DEPS[@]}"
		do
		echo -n " Direct GekkoFS dependencies: "
		for d in "${DIRECT_DEPS[@]}"; do
		echo -n "$d "
		done
		echo ""

		echo -n " fh2: "
		for d in "${FH2_DEPS[@]}"
		do
		echo -n " All: "
		for d in "${ALL_DEPS[@]}"; do
		echo -n "$d "
		done
		echo ""
		}

		check_dependency() {
		local DEP=$1
		shift
		local DEP_CONFIG=("$@")
		# ignore template when specific dependency is set
		if [[ -n "${DEPENDENCY}" ]]; then
		# check if specific dependency was set and return from function
		if echo "${DEPENDENCY}" \| grep -q "${DEP}"; then
		return
		fi
		# [[ "${DEPENDENCY}" == "${DEP}" ]] && return
		else
		# if not check if dependency is part of dependency config
		for e in "${DEP_CONFIG[@]}"; do
		if [[ "${DEP}" == "${e}" ]]; then
		return
		fi
		done
		fi
		false
		}

		prepare_build_dir() {
		if [ ! -d "$1/build" ]; then
		mkdir $1/build
		mkdir "$1"/build
		fi
		rm -rf $1/build/*
		rm -rf "$1"/build/*
		}

		find_cmake() {
		local CMAKE=`command -v cmake3 \|\| command -v cmake`
		local CMAKE
		CMAKE=$(command -v cmake3 \|\| command -v cmake)
		if [ $? -ne 0 ]; then
		>&2 echo "ERROR: could not find cmake"
		echo >&2 "ERROR: could not find cmake"
		exit 1
		fi
		echo ${CMAKE}
		echo "${CMAKE}"
		}

		PATCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
		PATCH_DIR="${PATCH_DIR}/patches"
		CLUSTER=""
		DEPENDENCY=""
		NA_LAYER=""
		CORES=""
		SOURCE=""
		INSTALL=""

		POSITIONAL=()
		while [[ $# -gt 0 ]]
		do
		while [[ $# -gt 0 ]]; do
		key="$1"

		case ${key} in
		@@ -116,14 +133,22 @@ case ${key} in
		shift # past argument
		shift # past value
		;;
		-c\|--cluster)
		CLUSTER="$2"
		-c \| --config)
		if [[ -z "$2" ]]; then
		echo "ERROR: Missing argument for -c/--config option"
		exit 1
		fi
		if ! echo "$VALID_DEP_OPTIONS" \| grep -q "$2"; then
		echo "ERROR: Invalid argument for -c/--config option"
		exit 1
		fi
		TMP_DEP_CONF="$2"
		shift # past argument
		shift # past value
		;;
		-d \| --dependency)
		if [[ -z "$2" ]]; then
		echo "Missing argument for -d/--dependency option"
		echo "ERROR: Missing argument for -d/--dependency option"
		exit
		fi
		DEPENDENCY="$2"
		@@ -158,7 +183,7 @@ set -- "${POSITIONAL[@]}" # restore positional parameters

		# deal with positional arguments
		if [[ (-z ${1+x}) \|\| (-z ${2+x}) ]]; then
		echo "Positional arguments missing."
		echo "ERROR: Positional arguments missing."
		usage_short
		exit 1
		fi
		@@ -166,16 +191,16 @@ SOURCE="$( readlink -mn "${1}" )"
		INSTALL="$(readlink -mn "${2}")"

		# deal with optional arguments
		if [ "${NA_LAYER}" == "" ]; then
		echo "Defaulting NAPLUGIN to 'all'"
		NA_LAYER="all"
		if [[ "${NA_LAYER}" == "" ]]; then
		echo "Defaulting NAPLUGIN to 'ofi'"
		NA_LAYER="ofi"
		fi
		if [ "${CORES}" == "" ]; then
		if [[ "${CORES}" == "" ]]; then
		CORES=$(grep -c ^processor /proc/cpuinfo)
		echo "CORES = ${CORES} (default)"
		else
		if [ ! "${CORES}" -gt "0" ]; then
		echo "CORES set to ${CORES} which is invalid.
		if [[ ! "${CORES}" -gt "0" ]]; then
		echo "ERROR: CORES set to ${CORES} which is invalid.
		Input must be numeric and greater than 0."
		usage_short
		exit
		@@ -183,35 +208,40 @@ Input must be numeric and greater than 0."
		echo CORES = "${CORES}"
		fi
		fi
		if [ "${NA_LAYER}" == "bmi" ] \|\| [ "${NA_LAYER}" == "ofi" ] \|\| [ "${NA_LAYER}" == "all" ]; then
		if [[ "${NA_LAYER}" == "bmi" \|\| "${NA_LAYER}" == "ofi" \|\| "${NA_LAYER}" == "all" ]]; then
		echo NAPLUGIN = "${NA_LAYER}"
		else
		echo "No valid plugin selected"
		echo "ERROR: No valid NA plugin selected"
		usage_short
		exit
		fi
		if [[ "${CLUSTER}" != "" ]]; then
		if [[ ( "${CLUSTER}" == "mogon1" ) \|\| ( "${CLUSTER}" == "fh2" ) \|\| ( "${CLUSTER}" == "mogon2" ) ]]; then
		echo CLUSTER = "${CLUSTER}"
		else
		echo "${CLUSTER} cluster configuration is invalid. Exiting ..."
		usage_short
		exit
		fi
		else
		echo "No cluster configuration set."
		fi
		# enable predefined dependency template
		case ${TMP_DEP_CONF} in
		mogon2)
		DEP_CONFIG=("${MOGON2_DEPS[@]}")
		echo "'Mogon2' dependencies are compiled"
		;;
		all)
		DEP_CONFIG=("${ALL_DEPS[@]}")
		echo "'All' dependencies are compiled"
		;;
		direct \| *)
		DEP_CONFIG=("${DIRECT_DEPS[@]}")
		echo "'Direct' GekkoFS dependencies are compiled (default)"
		;;
		esac

		USE_BMI="-DNA_USE_BMI:BOOL=OFF"
		USE_OFI="-DNA_USE_OFI:BOOL=OFF"

		CMAKE=`find_cmake`
		CMAKE=$(find_cmake)
		CMAKE="${CMAKE} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}"

		echo "Source path = ${SOURCE}";
		echo "Install path = ${INSTALL}";
		echo "Source path = ${SOURCE}"
		echo "Install path = ${INSTALL}"
		echo "------------------------------------"

		mkdir -p ${SOURCE}
		mkdir -p "${SOURCE}"

		######### From now on exits on any error ########
		set -e
		@@ -219,64 +249,72 @@ set -e
		export CPATH="${CPATH}:${INSTALL}/include"
		export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL}/lib:${INSTALL}/lib64"

		## Third party dependencies

		# Set cluster dependencies first
		if [[ ( "${CLUSTER}" == "mogon1" ) \|\| ( "${CLUSTER}" == "fh2" ) \|\| ( "${CLUSTER}" == "mogon2" ) ]]; then

		# compile zstd
		if [[ ( "${DEPENDENCY}" == "" ) \|\| ( "${DEPENDENCY}" == "zstd" ) ]]; then
		# build zstd for fast compression in rocksdb
		if check_dependency "zstd" "${DEP_CONFIG[@]}"; then
		echo "############################################################ Installing: zstd"
		CURR=${SOURCE}/zstd/build/cmake
		prepare_build_dir ${CURR}
		cd ${CURR}/build
		$CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Release ..
		make -j${CORES}
		prepare_build_dir "${CURR}"
		cd "${CURR}"/build
		$CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Release ..
		make -j"${CORES}"
		make install
		fi

		# build lz4
		if [[ ( "${DEPENDENCY}" == "" ) \|\| ( "${DEPENDENCY}" == "zstd" ) ]]; then
		echo "############################################################ Installing: zstd"
		# build zlib for rocksdb
		if check_dependency "lz4" "${DEP_CONFIG[@]}"; then
		echo "############################################################ Installing: lz4"
		CURR=${SOURCE}/lz4
		cd ${CURR}
		make -j${CORES}
		make DESTDIR=${INSTALL} PREFIX="" install
		cd "${CURR}"
		make -j"${CORES}"
		make DESTDIR="${INSTALL}" PREFIX="" install
		fi

		# build snappy
		if [[ ( "${DEPENDENCY}" == "" ) \|\| ( "${DEPENDENCY}" == "snappy" ) ]]; then
		# build snappy for rocksdb
		if check_dependency "snappy" "${DEP_CONFIG[@]}"; then
		echo "############################################################ Installing: snappy"
		CURR=${SOURCE}/snappy
		prepare_build_dir ${CURR}
		cd ${CURR}/build
		$CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Release ..
		make -j${CORES}
		prepare_build_dir "${CURR}"
		cd "${CURR}"/build
		$CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Release ..
		make -j"${CORES}"
		make install
		fi

		# build capstone for syscall-intercept
		if check_dependency "capstone" "${DEP_CONFIG[@]}"; then
		echo "############################################################ Installing: capstone"
		CURR=${SOURCE}/capstone
		prepare_build_dir "${CURR}"
		cd "${CURR}"/build
		$CMAKE -DCMAKE_INSTALL_PREFIX=/home/vef/gekkofs_deps/install -DCMAKE_BUILD_TYPE:STRING=Release ..
		make -j"${CORES}" install
		fi

		# build bmi
		if [[ ( "${DEPENDENCY}" == "" ) \|\| ( "${DEPENDENCY}" == "bmi" ) ]]; then
		if [ "$NA_LAYER" == "bmi" ] \|\| [ "$NA_LAYER" == "all" ]; then
		# build BMI
		if check_dependency "bmi" "${DEP_CONFIG[@]}"; then
		if [[ "${NA_LAYER}" == "bmi" \|\| "${NA_LAYER}" == "all" ]]; then
		USE_BMI="-DNA_USE_BMI:BOOL=ON"
		echo "############################################################ Installing: BMI"
		# BMI
		CURR=${SOURCE}/bmi
		prepare_build_dir ${CURR}
		cd ${CURR}
		prepare_build_dir "${CURR}"
		cd "${CURR}"
		./prepare
		cd ${CURR}/build
		CFLAGS="${CFLAGS} -w" ../configure --prefix=${INSTALL} --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict
		make -j${CORES}
		cd "${CURR}"/build
		CFLAGS="${CFLAGS} -w" ../configure --prefix="${INSTALL}" --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict
		make -j"${CORES}"
		make install
		fi
		fi

		# build ofi
		if [[ ( "${DEPENDENCY}" == "" ) \|\| ( "${DEPENDENCY}" == "ofi" ) ]]; then
		if [ "$NA_LAYER" == "ofi" ] \|\| [ "$NA_LAYER" == "all" ]; then
		if check_dependency "ofi" "${DEP_CONFIG[@]}"; then
		if [[ "${NA_LAYER}" == "ofi" \|\| "${NA_LAYER}" == "all" ]]; then
		USE_OFI="-DNA_USE_OFI:BOOL=ON"
		# Mogon2 already has libfabric installed in a version that Mercury supports.
		if [[ ("${CLUSTER}" != "mogon2") ]]; then
		echo "############################################################ Installing: LibFabric"
		#libfabric
		CURR=${SOURCE}/libfabric
		@@ -288,24 +326,22 @@ if [[ ( "${DEPENDENCY}" == "" ) \|\| ( "${DEPENDENCY}" == "ofi" ) ]]; then
		[ "${PERFORM_TEST}" ] && make check
		fi
		fi
		fi


		# Mercury
		if [[ ( "${DEPENDENCY}" == "" ) \|\| ( "${DEPENDENCY}" == "mercury" ) ]]; then
		if check_dependency "mercury" "${DEP_CONFIG[@]}"; then

		if [ "$NA_LAYER" == "bmi" ] \|\| [ "$NA_LAYER" == "all" ]; then
		if [[ "${NA_LAYER}" == "bmi" \|\| "${NA_LAYER}" == "all" ]]; then
		USE_BMI="-DNA_USE_BMI:BOOL=ON"
		fi

		if [ "$NA_LAYER" == "ofi" ] \|\| [ "$NA_LAYER" == "all" ]; then
		if [[ "${NA_LAYER}" == "ofi" \|\| "${NA_LAYER}" == "all" ]]; then
		USE_OFI="-DNA_USE_OFI:BOOL=ON"
		fi

		echo "############################################################ Installing: Mercury"
		CURR=${SOURCE}/mercury
		prepare_build_dir ${CURR}
		cd ${CURR}/build
		prepare_build_dir "${CURR}"
		cd "${CURR}"/build
		PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig $CMAKE \
		-DCMAKE_BUILD_TYPE:STRING=Release \
		-DBUILD_TESTING:BOOL=ON \
		@@ -318,67 +354,66 @@ if [[ ( "${DEPENDENCY}" == "" ) \|\| ( "${DEPENDENCY}" == "mercury" ) ]]; then
		-DCMAKE_INSTALL_PREFIX=${INSTALL} \
		${USE_BMI} ${USE_OFI} \
		..
		make -j${CORES}
		make -j"${CORES}"
		make install
		fi

		# Argobots
		if [[ ( "${DEPENDENCY}" == "" ) \|\| ( "${DEPENDENCY}" == "argobots" ) ]]; then
		if check_dependency "argobots" "${DEP_CONFIG[@]}"; then
		echo "############################################################ Installing: Argobots"
		CURR=${SOURCE}/argobots
		prepare_build_dir ${CURR}
		cd ${CURR}
		prepare_build_dir "${CURR}"
		cd "${CURR}"
		./autogen.sh
		cd ${CURR}/build
		../configure --prefix=${INSTALL} --enable-perf-opt --disable-checks
		make -j${CORES}
		cd "${CURR}"/build
		../configure --prefix="${INSTALL}" --enable-perf-opt --disable-checks
		make -j"${CORES}"
		make install
		[ "${PERFORM_TEST}" ] && make check
		fi

		# Margo
		if [[ ( "${DEPENDENCY}" == "" ) \|\| ( "${DEPENDENCY}" == "margo" ) ]]; then
		if check_dependency "margo" "${DEP_CONFIG[@]}"; then
		echo "############################################################ Installing: Margo"
		CURR=${SOURCE}/margo
		prepare_build_dir ${CURR}
		cd ${CURR}
		prepare_build_dir "${CURR}"
		cd "${CURR}"
		./prepare.sh
		cd ${CURR}/build
		../configure --prefix=${INSTALL} PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig CFLAGS="${CFLAGS} -Wall -O3"
		make -j${CORES}
		cd "${CURR}"/build
		../configure --prefix="${INSTALL}" PKG_CONFIG_PATH="${INSTALL}"/lib/pkgconfig CFLAGS="${CFLAGS} -Wall -O3"
		make -j"${CORES}"
		make install
		[ "${PERFORM_TEST}" ] && make check
		fi

		# Rocksdb
		if [[ ( "${DEPENDENCY}" == "" ) \|\| ( "${DEPENDENCY}" == "rocksdb" ) ]]; then
		if check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then
		echo "############################################################ Installing: Rocksdb"
		CURR=${SOURCE}/rocksdb
		cd ${CURR}
		cd "${CURR}"
		make clean
		USE_RTTI=1 make -j${CORES} static_lib
		INSTALL_PATH=${INSTALL} make install
		USE_RTTI=1 make -j"${CORES}" static_lib
		INSTALL_PATH="${INSTALL}" make install
		fi

		# syscall_intercept
		if [[ ( "${DEPENDENCY}" == "" ) \|\| ( "${DEPENDENCY}" == "syscall_intercept" ) ]]; then
		if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then
		echo "############################################################ Installing: Syscall_intercept"
		CURR=${SOURCE}/syscall_intercept
		prepare_build_dir ${CURR}
		cd ${CURR}/build
		$CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF ..
		prepare_build_dir "${CURR}"
		cd "${CURR}"/build
		$CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF ..
		make install
		fi

		# date
		if [[ ( "${DEPENDENCY}" == "" ) \|\| ( "${DEPENDENCY}" == "date" ) ]]; then
		if check_dependency "date" "${DEP_CONFIG[@]}"; then
		echo "############################################################ Installing: date"
		CURR=${SOURCE}/date
		prepare_build_dir ${CURR}
		cd ${CURR}/build
		$CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON ..
		prepare_build_dir "${CURR}"
		cd "${CURR}"/build
		$CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON ..
		make install
		fi


		echo "Done"

scripts/dl_dep.sh

+179 −138

File changed.

Preview size limit exceeded, changes collapsed.