Resolve "[Mogon II] Fix dependency compilation scripts"

Marc Vefrequested to merge
45-mogon-ii-fix-dependency-compilation-scripts into master

Closes #45 (closed)

Added Mogon2 dependencies and some script maintenance.

Edited by Marc Vef

Merge request reports

scripts/compile_dep.sh

+243 −208
Changes for scripts/compile_dep.sh: 243 added lines, 208 removed lines.
Original line number Diff line number Diff line 
#!/bin/bash
 








MOGON1_DEPS=( 









    "zstd" "lz4" "snappy" "bmi" "libfabric" "mercury" "argobots" "margo" 









    "rocksdb" "syscall_intercept date"









)









PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"








PATCH_DIR="${PATCH_DIR}/patches"








DEPENDENCY=""








NA_LAYER=""








CORES=""








SOURCE=""








INSTALL=""








DEP_CONFIG=""
















VALID_DEP_OPTIONS="mogon2 direct all"
















MOGON2_DEPS=(








    "zstd" "lz4" "snappy" "bmi" "mercury" "argobots" "margo" "rocksdb" 









    "syscall_intercept date"









    "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"








    "syscall_intercept" "date"








)
















DIRECT_DEPS=(








  "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date"








)
















FH2_DEPS=(









    "zstd" "lz4" "snappy" "bmi" "libfabric" "mercury" "argobots" "margo" 









    "rocksdb" "syscall_intercept date"









ALL_DEPS=(








    "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb"








     "syscall_intercept" "date"








)
















usage_short() {








	echo "








usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CLUSTER>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>]









usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CONFIG>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>]








                      source_path install_path








	"








}
















help_msg() {
















	usage_short









    usage_short








    echo "








This script compiles all GekkoFS dependencies (excluding the fs itself)

















@@ -40,12 +50,14 @@ optional arguments:







    -n <NAPLUGIN>, --na <NAPLUGIN>








                network layer that is used for communication. Valid: {bmi,ofi,all}








                defaults to 'all'








    -c <CLUSTER>, --cluster <CLUSTER>









                additional configurations for specific compute clusters









                supported clusters: {mogon1,mogon2,fh2}









    -c <CONFIG>, --config <CONFIG>








                allows additional configurations, e.g., for specific clusters








                supported values: {mogon2, direct, all}








                defaults to 'direct'








    -d <DEPENDENCY>, --dependency <DEPENDENCY>








                build and install a specific dependency. If unspecified 









                all dependencies are built and installed.









                download a specific dependency and ignore --config setting. If unspecified








                all dependencies are built and installed based on set --config setting.








                Multiple dependencies are supported: Pass a space-separated string (e.g., \"ofi mercury\"








    -j <COMPILE_CORES>, --compilecores <COMPILE_CORES>








                number of cores that are used to compile the dependencies








                defaults to number of available cores









@@ -53,165 +65,183 @@ optional arguments:







"








}


























list_dependencies() {
















    echo "Available dependencies: "
















    echo -n "  Mogon 1: "









    for d in "${MOGON1_DEPS[@]}"









    do









    echo -n "  Mogon 2: "








    for d in "${MOGON2_DEPS[@]}"; do








        echo -n "$d "








    done








    echo ""



















    echo -n "  Mogon 2: "









    for d in "${MOGON2_DEPS[@]}"









    do









    echo -n "  Direct GekkoFS dependencies: "








    for d in "${DIRECT_DEPS[@]}"; do








        echo -n "$d "








    done








    echo ""



















    echo -n "  fh2: "









    for d in "${FH2_DEPS[@]}"









    do









    echo -n "  All: "








    for d in "${ALL_DEPS[@]}"; do








        echo -n "$d "








    done








    echo ""








}
















check_dependency() {








  local DEP=$1








  shift








  local DEP_CONFIG=("$@")








  # ignore template when specific dependency is set








  if [[ -n "${DEPENDENCY}" ]]; then








      # check if specific dependency was set and return from function








      if echo "${DEPENDENCY}" | grep -q "${DEP}"; then








        return








      fi








#      [[ "${DEPENDENCY}" == "${DEP}" ]] && return








  else








      # if not check if dependency is part of dependency config








      for e in "${DEP_CONFIG[@]}"; do








        if [[ "${DEP}" == "${e}" ]]; then








          return








        fi








      done








  fi








  false








}
















prepare_build_dir() {








    if [ ! -d "$1/build" ]; then








        mkdir $1/build









        mkdir "$1"/build








    fi








    rm -rf $1/build/*









    rm -rf "$1"/build/*








}
















find_cmake() {








    local CMAKE=`command -v cmake3 || command -v cmake`









    local CMAKE








    CMAKE=$(command -v cmake3 || command -v cmake)








    if [ $? -ne 0 ]; then








        >&2 echo "ERROR: could not find cmake"









        echo >&2 "ERROR: could not find cmake"








        exit 1








    fi








    echo ${CMAKE}









    echo "${CMAKE}"








}
















PATCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"









PATCH_DIR="${PATCH_DIR}/patches"









CLUSTER=""









DEPENDENCY=""









NA_LAYER=""









CORES=""









SOURCE=""









INSTALL=""



















POSITIONAL=()








while [[ $# -gt 0 ]]









do









key="$1"



















case ${key} in









    -n|--na)









    NA_LAYER="$2"









    shift # past argument









    shift # past value









    ;;









	-c|--cluster)









    CLUSTER="$2"









    shift # past argument









    shift # past value









    ;;









    -d|--dependency)









    if [[ -z "$2" ]]; then









        echo "Missing argument for -d/--dependency option"









while [[ $# -gt 0 ]]; do








    key="$1"
















    case ${key} in








    -n | --na)








        NA_LAYER="$2"








        shift # past argument








        shift # past value








        ;;








    -c | --config)








        if [[ -z "$2" ]]; then








            echo "ERROR: Missing argument for -c/--config option"








            exit 1








        fi








        if ! echo "$VALID_DEP_OPTIONS" | grep -q "$2"; then








            echo "ERROR: Invalid argument for -c/--config option"








            exit 1








        fi








        TMP_DEP_CONF="$2"








        shift # past argument








        shift # past value








        ;;








    -d | --dependency)








        if [[ -z "$2" ]]; then








            echo "ERROR: Missing argument for -d/--dependency option"








            exit








        fi








        DEPENDENCY="$2"








        shift # past argument








        shift # past value








        ;;








    -j | --compilecores)








        CORES="$2"








        shift # past argument








        shift # past value








        ;;








    -t | --test)








        PERFORM_TEST=true








        shift








        ;;








    -l | --list-dependencies)








        list_dependencies








        exit








    fi









    DEPENDENCY="$2"









    shift # past argument









    shift # past value









    ;;









	-j|--compilecores)









    CORES="$2"









    shift # past argument









    shift # past value









    ;;









    -t|--test)









    PERFORM_TEST=true









    shift









    ;;









    -l|--list-dependencies)









    list_dependencies









    exit









    ;;









    -h|--help)









    help_msg









	exit









    #shift # past argument









    ;;









    *)    # unknown option









    POSITIONAL+=("$1") # save it in an array for later









    shift # past argument









    ;;









esac









        ;;








    -h | --help)








        help_msg








        exit








        #shift # past argument








        ;;








    *) # unknown option








        POSITIONAL+=("$1") # save it in an array for later








        shift              # past argument








        ;;








    esac








done








set -- "${POSITIONAL[@]}" # restore positional parameters
















# deal with positional arguments








if [[ ( -z ${1+x} ) || ( -z ${2+x} ) ]]; then









    echo "Positional arguments missing."









if [[ (-z ${1+x}) || (-z ${2+x}) ]]; then








    echo "ERROR: Positional arguments missing."








    usage_short








    exit 1








fi








SOURCE="$( readlink -mn "${1}" )"









INSTALL="$( readlink -mn "${2}" )"









SOURCE="$(readlink -mn "${1}")"








INSTALL="$(readlink -mn "${2}")"
















# deal with optional arguments








if [ "${NA_LAYER}" == "" ]; then









	echo "Defaulting NAPLUGIN to 'all'"









	NA_LAYER="all"









if [[ "${NA_LAYER}" == "" ]]; then








    echo "Defaulting NAPLUGIN to 'ofi'"








    NA_LAYER="ofi"








fi








if [ "${CORES}" == "" ]; then









	CORES=$(grep -c ^processor /proc/cpuinfo)









	echo "CORES = ${CORES} (default)"









if [[ "${CORES}" == "" ]]; then








    CORES=$(grep -c ^processor /proc/cpuinfo)








    echo "CORES = ${CORES} (default)"








else








	if [ ! "${CORES}" -gt "0" ]; then









		echo "CORES set to ${CORES} which is invalid.









    if [[ ! "${CORES}" -gt "0" ]]; then








        echo "ERROR: CORES set to ${CORES} which is invalid.








Input must be numeric and greater than 0."








		usage_short









		exit









	else









		echo CORES    = "${CORES}"









	fi









fi









if [ "${NA_LAYER}" == "bmi" ] || [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then









	echo NAPLUGIN = "${NA_LAYER}"









else









    echo "No valid plugin selected"









    usage_short









    exit









fi









if [[ "${CLUSTER}" != "" ]]; then









	if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) || ( "${CLUSTER}" == "mogon2" ) ]]; then









		echo CLUSTER  = "${CLUSTER}"









    else









        echo "${CLUSTER} cluster configuration is invalid. Exiting ..."









        usage_short








        exit








    else








        echo CORES = "${CORES}"








    fi








fi








if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then








    echo NAPLUGIN = "${NA_LAYER}"








else








    echo "No cluster configuration set."









    echo "ERROR: No valid NA plugin selected"








    usage_short








    exit








fi








# enable predefined dependency template








case ${TMP_DEP_CONF} in








mogon2)








  DEP_CONFIG=("${MOGON2_DEPS[@]}")








  echo "'Mogon2' dependencies are compiled"








  ;;








all)








  DEP_CONFIG=("${ALL_DEPS[@]}")








  echo "'All' dependencies are compiled"








  ;;








direct | *)








  DEP_CONFIG=("${DIRECT_DEPS[@]}")








  echo "'Direct' GekkoFS dependencies are compiled (default)"








  ;;








esac
















USE_BMI="-DNA_USE_BMI:BOOL=OFF"








USE_OFI="-DNA_USE_OFI:BOOL=OFF"
















CMAKE=`find_cmake`









CMAKE=$(find_cmake)








CMAKE="${CMAKE} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}"
















echo "Source path = ${SOURCE}";









echo "Install path = ${INSTALL}";









echo "Source path = ${SOURCE}"








echo "Install path = ${INSTALL}"








echo "------------------------------------"
















mkdir -p ${SOURCE}









mkdir -p "${SOURCE}"
















######### From now on exits on any error ########








set -e









@@ -219,93 +249,99 @@ set -e







export CPATH="${CPATH}:${INSTALL}/include"








export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL}/lib:${INSTALL}/lib64"
















## Third party dependencies
















# Set cluster dependencies first








if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) || ( "${CLUSTER}" == "mogon2" ) ]]; then

















    # compile zstd









    if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "zstd" ) ]]; then









        echo "############################################################ Installing:  zstd"









        CURR=${SOURCE}/zstd/build/cmake









        prepare_build_dir ${CURR}









        cd ${CURR}/build









        $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Release ..









        make -j${CORES}









        make install









    fi









# build zstd for fast compression in rocksdb








if check_dependency "zstd" "${DEP_CONFIG[@]}"; then








    echo "############################################################ Installing:  zstd"








    CURR=${SOURCE}/zstd/build/cmake








    prepare_build_dir "${CURR}"








    cd "${CURR}"/build








    $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Release ..








    make -j"${CORES}"








    make install








fi
















    # build lz4









    if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "zstd" ) ]]; then









        echo "############################################################ Installing:  zstd"









        CURR=${SOURCE}/lz4









        cd ${CURR}









        make -j${CORES}









        make DESTDIR=${INSTALL} PREFIX="" install









    fi









# build zlib for rocksdb








if check_dependency "lz4" "${DEP_CONFIG[@]}"; then








    echo "############################################################ Installing:  lz4"








    CURR=${SOURCE}/lz4








    cd "${CURR}"








    make -j"${CORES}"








    make DESTDIR="${INSTALL}" PREFIX="" install








fi
















    # build snappy









    if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "snappy" ) ]]; then









        echo "############################################################ Installing:  snappy"









        CURR=${SOURCE}/snappy









        prepare_build_dir ${CURR}









        cd ${CURR}/build









        $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Release ..









        make -j${CORES}









        make install









    fi









# build snappy for rocksdb








if check_dependency "snappy" "${DEP_CONFIG[@]}"; then








    echo "############################################################ Installing:  snappy"








    CURR=${SOURCE}/snappy








    prepare_build_dir "${CURR}"








    cd "${CURR}"/build








    $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Release ..








    make -j"${CORES}"








    make install








fi
















# build capstone for syscall-intercept








if check_dependency "capstone" "${DEP_CONFIG[@]}"; then








    echo "############################################################ Installing:  capstone"








    CURR=${SOURCE}/capstone








    prepare_build_dir "${CURR}"








    cd "${CURR}"/build








    $CMAKE -DCMAKE_INSTALL_PREFIX=/home/vef/gekkofs_deps/install -DCMAKE_BUILD_TYPE:STRING=Release ..








    make -j"${CORES}" install








fi
















# build bmi









if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "bmi" ) ]]; then









    if [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "all" ]; then









# build BMI








if check_dependency "bmi" "${DEP_CONFIG[@]}"; then








    if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "all" ]]; then








        USE_BMI="-DNA_USE_BMI:BOOL=ON"








        echo "############################################################ Installing:  BMI"








        # BMI








        CURR=${SOURCE}/bmi








        prepare_build_dir ${CURR}









        cd ${CURR}









        prepare_build_dir "${CURR}"








        cd "${CURR}"








        ./prepare








        cd ${CURR}/build









        CFLAGS="${CFLAGS} -w" ../configure --prefix=${INSTALL} --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict









        make -j${CORES}









        cd "${CURR}"/build








        CFLAGS="${CFLAGS} -w" ../configure --prefix="${INSTALL}" --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict








        make -j"${CORES}"








        make install








    fi








fi
















# build ofi








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "ofi" ) ]]; then









    if [ "$NA_LAYER" == "ofi" ] || [ "$NA_LAYER" == "all" ]; then









if check_dependency "ofi" "${DEP_CONFIG[@]}"; then








    if [[ "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then








        USE_OFI="-DNA_USE_OFI:BOOL=ON"








        # Mogon2 already has libfabric installed in a version that Mercury supports.









        if [[ ("${CLUSTER}" != "mogon2") ]]; then









            echo "############################################################ Installing:  LibFabric"









            #libfabric









            CURR=${SOURCE}/libfabric









            prepare_build_dir ${CURR}









            cd ${CURR}/build









            ../configure --prefix=${INSTALL} --enable-tcp=yes









            make -j${CORES}









            make install









            [ "${PERFORM_TEST}" ] && make check









        fi









        echo "############################################################ Installing:  LibFabric"








        #libfabric








        CURR=${SOURCE}/libfabric








        prepare_build_dir ${CURR}








        cd ${CURR}/build








        ../configure --prefix=${INSTALL} --enable-tcp=yes








        make -j${CORES}








        make install








        [ "${PERFORM_TEST}" ] && make check








    fi








fi


























# Mercury








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "mercury" ) ]]; then









if check_dependency "mercury" "${DEP_CONFIG[@]}"; then
















    if [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "all" ]; then









    if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "all" ]]; then








        USE_BMI="-DNA_USE_BMI:BOOL=ON"








    fi
















    if [ "$NA_LAYER" == "ofi" ] || [ "$NA_LAYER" == "all" ]; then









    if [[ "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then








        USE_OFI="-DNA_USE_OFI:BOOL=ON"








    fi
















    echo "############################################################ Installing:  Mercury"








    CURR=${SOURCE}/mercury








    prepare_build_dir ${CURR}









    cd ${CURR}/build









    prepare_build_dir "${CURR}"








    cd "${CURR}"/build








    PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig $CMAKE \








        -DCMAKE_BUILD_TYPE:STRING=Release \








        -DBUILD_TESTING:BOOL=ON \









@@ -318,67 +354,66 @@ if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "mercury" ) ]]; then







        -DCMAKE_INSTALL_PREFIX=${INSTALL} \








        ${USE_BMI} ${USE_OFI} \








        ..








    make -j${CORES}









    make -j"${CORES}"








    make install








fi
















# Argobots








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "argobots" ) ]]; then









if check_dependency "argobots" "${DEP_CONFIG[@]}"; then








    echo "############################################################ Installing:  Argobots"








    CURR=${SOURCE}/argobots








    prepare_build_dir ${CURR}









    cd ${CURR}









    prepare_build_dir "${CURR}"








    cd "${CURR}"








    ./autogen.sh








    cd ${CURR}/build









    ../configure --prefix=${INSTALL} --enable-perf-opt --disable-checks









    make -j${CORES}









    cd "${CURR}"/build








    ../configure --prefix="${INSTALL}" --enable-perf-opt --disable-checks








    make -j"${CORES}"








    make install








    [ "${PERFORM_TEST}" ] && make check








fi
















# Margo








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "margo" ) ]]; then









if check_dependency "margo" "${DEP_CONFIG[@]}"; then








    echo "############################################################ Installing:  Margo"








    CURR=${SOURCE}/margo








    prepare_build_dir ${CURR}









    cd ${CURR}









    prepare_build_dir "${CURR}"








    cd "${CURR}"








    ./prepare.sh








    cd ${CURR}/build









    ../configure --prefix=${INSTALL} PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig CFLAGS="${CFLAGS} -Wall -O3"









    make -j${CORES}









    cd "${CURR}"/build








    ../configure --prefix="${INSTALL}" PKG_CONFIG_PATH="${INSTALL}"/lib/pkgconfig CFLAGS="${CFLAGS} -Wall -O3"








    make -j"${CORES}"








    make install








    [ "${PERFORM_TEST}" ] && make check








fi
















# Rocksdb








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "rocksdb" ) ]]; then









if check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then








    echo "############################################################ Installing:  Rocksdb"








    CURR=${SOURCE}/rocksdb








    cd ${CURR}









    cd "${CURR}"








    make clean








    USE_RTTI=1 make -j${CORES} static_lib









    INSTALL_PATH=${INSTALL} make install









    USE_RTTI=1 make -j"${CORES}" static_lib








    INSTALL_PATH="${INSTALL}" make install








fi
















# syscall_intercept








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "syscall_intercept" ) ]]; then









if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then








    echo "############################################################ Installing:  Syscall_intercept"








    CURR=${SOURCE}/syscall_intercept








    prepare_build_dir ${CURR}









    cd ${CURR}/build









    $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF ..









    prepare_build_dir "${CURR}"








    cd "${CURR}"/build








    $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF ..








    make install








fi
















# date








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "date" ) ]]; then









if check_dependency "date" "${DEP_CONFIG[@]}"; then








    echo "############################################################ Installing:  date"








    CURR=${SOURCE}/date








    prepare_build_dir ${CURR}









    cd ${CURR}/build









    $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON ..









    prepare_build_dir "${CURR}"








    cd "${CURR}"/build








    $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON ..








    make install








fi


























echo "Done"
























scripts/dl_dep.sh



+179
−138








































Changes for scripts/dl_dep.sh: 179 added lines, 138 removed lines.



Original line number
Diff line number
Diff line









#!/bin/bash
















#set -x



















COMMON_CURL_FLAGS="--silent --fail --show-error --location -O"








COMMON_GIT_FLAGS="--quiet --single-branch"








PATCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"









PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"








PATCH_DIR="${PATCH_DIR}/patches"








CLUSTER=""









DEPENDENCY=""








NA_LAYER=""








DEP_CONFIG=""








VERBOSE=false
















MOGON1_DEPS=( 









    "zstd" "lz4" "snappy" "bmi" "libfabric" "mercury" "argobots" "margo" 









    "rocksdb" "syscall_intercept date"









)









VALID_DEP_OPTIONS="mogon2 direct all"
















MOGON2_DEPS=(








    "zstd" "lz4" "snappy" "bmi" "mercury" "argobots" "margo" "rocksdb" 









    "syscall_intercept date"









    "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"








    "syscall_intercept" "date"








)
















FH2_DEPS=(









    "zstd" "lz4" "snappy" "bmi" "libfabric" "mercury" "argobots" "margo" 









    "rocksdb" "syscall_intercept date"









DIRECT_DEPS=(








  "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date"








)
















ALL_DEPS=(








    "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb"








     "syscall_intercept" "date"








)
















# Stop all backround jobs on interruption.








# "kill -- -$$" sends a SIGTERM to the whole process group,








# thus killing also descendants.








trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM



















# Use single quotes, otherwise this expands now rather than when signalled.








# See shellcheck SC2064.








trap 'trap - SIGTERM && kill -- -$$' SIGINT SIGTERM
















exit_child() {








    if [ ! $? -eq 0 ]; then









@@ -40,38 +40,57 @@ exit_child() {







}
















error_exit() {








    echo "$1" >&2   ## Send message to stderr. Exclude >&2 if you don't want it that way.









    exit "${2:-1}"  ## Return a code specified by $2 or 1 by default.









    echo "$1" >&2  ## Send message to stderr. Exclude >&2 if you don't want it that way.








    exit "${2:-1}" ## Return a code specified by $2 or 1 by default.








}
















list_dependencies() {
















    echo "Available dependencies: "
















    echo -n "  Mogon 1: "









    for d in "${MOGON1_DEPS[@]}"









    do









    echo -n "  Mogon 2: "








    for d in "${MOGON2_DEPS[@]}"; do








        echo -n "$d "








    done








    echo ""



















    echo -n "  Mogon 2: "









    for d in "${MOGON2_DEPS[@]}"









    do









    echo -n "  Direct GekkoFS dependencies: "








    for d in "${DIRECT_DEPS[@]}"; do








        echo -n "$d "








    done








    echo ""



















    echo -n "  fh2: "









    for d in "${FH2_DEPS[@]}"









    do









    echo -n "  All: "








    for d in "${ALL_DEPS[@]}"; do








        echo -n "$d "








    done








    echo ""








}
















check_dependency() {








  local DEP=$1








  shift








  local DEP_CONFIG=("$@")








  # ignore template when specific dependency is set








  if [[ -n "${DEPENDENCY}" ]]; then








      # check if specific dependency was set and return from function








      if echo "${DEPENDENCY}" | grep -q "${DEP}"; then








        return








      fi








#      [[ "${DEPENDENCY}" == "${DEP}" ]] && return








  else








      # if not check if dependency is part of dependency config








      for e in "${DEP_CONFIG[@]}"; do








        if [[ "${DEP}" == "${e}" ]]; then








          return








        fi








      done








  fi








  false








}
















clonedeps() {








    set -ex









    if [[ "$VERBOSE" == true ]]; then








        set -ex








    else








        set -e








    fi








    trap exit_child EXIT
















    local FOLDER=$1









@@ -82,8 +101,8 @@ clonedeps() {















    local ACTION
















    if [ -d "${SOURCE}/${FOLDER}/.git" ]; then









        cd ${SOURCE}/${FOLDER} && git fetch -q









    if [[ -d "${SOURCE}/${FOLDER}/.git" ]]; then








        cd "${SOURCE}/${FOLDER}" && git fetch -q








        ACTION="Pulled"








    else








        git clone ${COMMON_GIT_FLAGS} ${GIT_FLAGS} -- "${REPO}" "${SOURCE}/${FOLDER}"









@@ -94,25 +113,29 @@ clonedeps() {







    echo "${ACTION} ${FOLDER} [$COMMIT]"
















    # apply patch if provided








    if [ ! -z ${PATCH} ]; then









        git apply --verbose ${PATCH_DIR}/${PATCH}









    if [[ -n "${PATCH}" ]]; then








        git apply --verbose "${PATCH_DIR}/${PATCH}"








    fi


















}
















wgetdeps() {








    set -e









    if [[ "$VERBOSE" == true ]]; then








        set -ex








    else








        set -e








    fi








    trap exit_child EXIT
















    FOLDER=$1








    URL=$2








    if [ -d "${SOURCE}/${FOLDER}" ]; then









        rm -rf "${SOURCE}/${FOLDER}"









    if [[ -d "${SOURCE}/${FOLDER}" ]]; then








        # SC2115 Use "${var:?}" to ensure this never expands to /* .








        rm -rf "${SOURCE:?}/${FOLDER:?}"








    fi








    mkdir -p "${SOURCE}/${FOLDER}"








    cd ${SOURCE}









    cd "${SOURCE}"








    FILENAME=$(basename $URL)








    if [ -f "${SOURCE}/$FILENAME" ]; then









    if [[ -f "${SOURCE}/$FILENAME" ]]; then








        rm -f "${SOURCE}/$FILENAME"








    fi








    curl ${COMMON_CURL_FLAGS} "$URL" || error_exit "Failed to download ${URL}" $?









@@ -123,14 +146,14 @@ wgetdeps() {















usage_short() {








	echo "








usage: dl_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CLUSTER>] [-d <DEPENDENCY>] 









usage: dl_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CONFIG>] [-d <DEPENDENCY>]








                    source_path








	"








}
















help_msg() {
















        usage_short









    usage_short








    echo "








This script gets all GekkoFS dependency sources (excluding the fs itself)

















@@ -141,167 +164,185 @@ positional arguments:







optional arguments:








        -h, --help              shows this help message and exits








        -l, --list-dependencies








                                list dependencies available for download









                                list dependencies available for download with descriptions








        -n <NAPLUGIN>, --na <NAPLUGIN>








                                network layer that is used for communication. Valid: {bmi,ofi,all}








                                defaults to 'all'









        -c <CLUSTER>, --cluster <CLUSTER>









                                additional configurations for specific compute clusters









                                supported clusters: {mogon1,mogon2,fh2}









                                defaults to 'ofi'








        -c <CONFIG>, --config <CONFIG>








                                allows additional configurations, e.g., for specific clusters








                                supported values: {mogon2, direct, all}








                                defaults to 'direct'








        -d <DEPENDENCY>, --dependency <DEPENDENCY>








                                download a specific dependency. If unspecified 









                                all dependencies are built and installed.









                                download a specific dependency and ignore --config setting. If unspecified








                                all dependencies are built and installed based on set --config setting.








                                Multiple dependencies are supported: Pass a space-separated string (e.g., \"ofi mercury\"








        -v, --verbose           Increase download verbosity








        "








}
















POSITIONAL=()








while [[ $# -gt 0 ]]









do









key="$1"



















case ${key} in









    -n|--na)









    NA_LAYER="$2"









    shift # past argument









    shift # past value









    ;;









    -c|--cluster)









    CLUSTER="$2"









    shift # past argument









    shift # past value









    ;;









    -d|--dependency)









    if [[ -z "$2" ]]; then









        echo "Missing argument for -d/--dependency option"









while [[ $# -gt 0 ]]; do








    key="$1"
















    case ${key} in








    -n | --na)








        NA_LAYER="$2"








        shift # past argument








        shift # past value








        ;;








    -c | --config)








        if [[ -z "$2" ]]; then








            echo "ERROR: Missing argument for -c/--config option"








            exit 1








        fi








        if ! echo "$VALID_DEP_OPTIONS" | grep -q "$2"; then








            echo "ERROR: Invalid argument for -c/--config option"








            exit 1








        fi








        TMP_DEP_CONF="$2"








        shift # past argument








        shift # past value








        ;;








    -d | --dependency)








        if [[ -z "$2" ]]; then








            echo "ERROR: Missing argument for -d/--dependency option"








            exit 1








        fi








        DEPENDENCY="$2"








        shift # past argument








        shift # past value








        ;;








    -l | --list-dependencies)








        list_dependencies








        exit








    fi









    DEPENDENCY="$2"









    shift # past argument









    shift # past value









    ;;









    -l|--list-dependencies)









    list_dependencies









    exit









    ;;









    -h|--help)









    help_msg









    exit









    #shift # past argument









    ;;









    *)    # unknown option









    POSITIONAL+=("$1") # save it in an array for later









    shift # past argument









    ;;









esac









        ;;








    -h | --help)








        help_msg








        exit








        ;;








    -v | --verbose)








        VERBOSE=true








        shift # past argument








        ;;








    *) # unknown option








        POSITIONAL+=("$1") # save it in an array for later








        shift              # past argument








        ;;








    esac








done








set -- "${POSITIONAL[@]}" # restore positional parameters
















# positional arguments








if [[ -z ${1+x} ]]; then








    echo "Positional arguments missing."









    echo "ERROR: Positional arguments missing."








    usage_short








    exit 1








fi








SOURCE="$( readlink -mn "${1}" )"









SOURCE="$(readlink -mn "${1}")"
















# optional arguments








if [ "${NA_LAYER}" == "" ]; then









        echo "Defaulting NAPLUGIN to 'all'"









        NA_LAYER="all"









if [[ "${NA_LAYER}" == "" ]]; then








    echo "Defaulting NAPLUGIN to 'ofi'"








    NA_LAYER="ofi"








fi
















# enable predefined dependency template








case ${TMP_DEP_CONF} in








mogon2)








  DEP_CONFIG=("${MOGON2_DEPS[@]}")








  [[ -z "${DEPENDENCY}" ]] && echo "'Mogon2' dependencies are downloaded"








  ;;








all)








  DEP_CONFIG=("${ALL_DEPS[@]}")








  [[ -z "${DEPENDENCY}" ]] && echo "'All' dependencies are downloaded"








  ;;








direct | *)








  DEP_CONFIG=("${DIRECT_DEPS[@]}")








  [[ -z "${DEPENDENCY}" ]] && echo "'Direct' GekkoFS dependencies are downloaded (default)"








  ;;








esac
















# sanity checks








if [[ ( "${NA_LAYER}" == "bmi" ) || ( "${NA_LAYER}" == "ofi" ) || ( "${NA_LAYER}" == "all" ) ]]; then









	echo NAPLUGIN = "${NA_LAYER}"









if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then








    echo NAPLUGIN = "${NA_LAYER}"








else








    echo "No valid plugin selected"









    echo "ERROR: No valid plugin selected"








    usage_short








    exit









fi









if [[ "${CLUSTER}" != "" ]]; then









	if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "mogon2" ) || ( "${CLUSTER}" == "fh2" ) ]]; then









		echo CLUSTER  = "${CLUSTER}"









    else









        echo "${CLUSTER} cluster configuration is invalid. Exiting ..."









        usage_short









        exit









    fi









else









    echo "No cluster configuration set."









    exit 1








fi
















echo "Source path is set to  \"${SOURCE}\""








echo "------------------------------------"
















mkdir -p ${SOURCE}









mkdir -p "${SOURCE}"
















# get cluster dependencies









if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "mogon2" ) || ( "${CLUSTER}" == "fh2" ) ]]; then









## Third party dependencies
















    # get zstd for fast compression in rocksdb









    if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "zstd" ) ]]; then









        wgetdeps "zstd" "https://github.com/facebook/zstd/archive/v1.3.2.tar.gz" &









    fi









# get zstd for fast compression in rocksdb








if check_dependency "zstd" "${DEP_CONFIG[@]}"; then








    wgetdeps "zstd" "https://github.com/facebook/zstd/archive/v1.3.2.tar.gz" &








fi
















    # get zlib for rocksdb









    if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "zstd" ) ]]; then









        wgetdeps "lz4" "https://github.com/lz4/lz4/archive/v1.8.0.tar.gz" &









    fi









# get zlib for rocksdb








if check_dependency "lz4" "${DEP_CONFIG[@]}"; then








    wgetdeps "lz4" "https://github.com/lz4/lz4/archive/v1.8.0.tar.gz" &








fi
















	# get snappy for rocksdb









    if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "snappy" ) ]]; then









            wgetdeps "snappy" "https://github.com/google/snappy/archive/1.1.7.tar.gz" &









    fi









# get snappy for rocksdb








if check_dependency "snappy" "${DEP_CONFIG[@]}"; then








    wgetdeps "snappy" "https://github.com/google/snappy/archive/1.1.7.tar.gz" &








fi








#if [ "${CLUSTER}" == "fh2" ]; then









	# no distinct 3rd party software needed as of now.









#fi

















# get capstone for syscall-intercept








if check_dependency "capstone" "${DEP_CONFIG[@]}"; then








    wgetdeps "capstone" "https://github.com/aquynh/capstone/archive/4.0.1.tar.gz" &








fi
















## Direct GekkoFS dependencies
















# get BMI








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "bmi" ) ]]; then









if check_dependency "bmi" "${DEP_CONFIG[@]}"; then








    if [ "${NA_LAYER}" == "bmi" ] || [ "${NA_LAYER}" == "all" ]; then








        clonedeps "bmi" "https://xgitlab.cels.anl.gov/sds/bmi.git" "81ad0575fc57a69269a16208417cbcbefa51f9ea" &








    fi








fi
















# get libfabric








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "ofi" ) ]]; then









if check_dependency "ofi" "${DEP_CONFIG[@]}"; then








    if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then








        # No need to get libfabric for mogon2 as it is already installed









        if [[ ("${CLUSTER}" != "mogon2") ]]; then









            wgetdeps "libfabric" "https://github.com/ofiwg/libfabric/releases/download/v1.8.1/libfabric-1.8.1.tar.bz2" &









        fi









        wgetdeps "libfabric" "https://github.com/ofiwg/libfabric/releases/download/v1.8.1/libfabric-1.8.1.tar.bz2" &








    fi








fi
















# get Mercury








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "mercury" ) ]]; then









if check_dependency "mercury" "${DEP_CONFIG[@]}"; then








    clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "fd410dfb9852b2b98d21113531f3058f45bfcd64"  "--recurse-submodules" &








fi
















# get Argobots








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "argobots" ) ]]; then









if check_dependency "argobots" "${DEP_CONFIG[@]}"; then








    wgetdeps "argobots" "https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz" &








fi
















# get Margo








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "margo" ) ]]; then









if check_dependency "margo" "${DEP_CONFIG[@]}"; then








    clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "016dbdce22da3fe4f97b46c20a53bced9370a217" &








fi
















# get rocksdb








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "rocksdb" ) ]]; then









if check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then








    wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz" &








fi
















# get syscall_intercept








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "syscall_intercept" ) ]]; then









if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then








    clonedeps "syscall_intercept" "https://github.com/pmem/syscall_intercept.git" "cc3412a2ad39f2e26cc307d5b155232811d7408e" "" "syscall_intercept.patch" &








fi
















# get date








if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "date" ) ]]; then









if check_dependency "date" "${DEP_CONFIG[@]}"; then








    clonedeps "date" "https://github.com/HowardHinnant/date.git" "e7e1482087f58913b80a20b04d5c58d9d6d90155" &








fi


















# Wait for all download to be completed








wait








echo "Done"








 No newline at end of file