Commit 1ec0d0fd authored by Alberto Miranda's avatar Alberto Miranda ♨️
Browse files

Merge branch '45-mogon-ii-fix-dependency-compilation-scripts' into 'master' 

Resolve "[Mogon II] Fix dependency compilation scripts"

Closes #45

See merge request hpc/gekkofs!25
parents 61cd8d98 55530639

scripts/compile_dep.sh

+171 −136
Original line number Diff line number Diff line 
#!/bin/bash



MOGON1_DEPS=( 

    "zstd" "lz4" "snappy" "bmi" "libfabric" "mercury" "argobots" "margo" 

    "rocksdb" "syscall_intercept date"

)

PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

PATCH_DIR="${PATCH_DIR}/patches"

DEPENDENCY=""

NA_LAYER=""

CORES=""

SOURCE=""

INSTALL=""

DEP_CONFIG=""



VALID_DEP_OPTIONS="mogon2 direct all"



MOGON2_DEPS=(

    "zstd" "lz4" "snappy" "bmi" "mercury" "argobots" "margo" "rocksdb" 

    "syscall_intercept date"

    "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb"

    "syscall_intercept" "date"

)



DIRECT_DEPS=(

  "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date"

)



FH2_DEPS=(

    "zstd" "lz4" "snappy" "bmi" "libfabric" "mercury" "argobots" "margo" 

    "rocksdb" "syscall_intercept date"

ALL_DEPS=(

    "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb"

     "syscall_intercept" "date"

)



usage_short() {

	echo "

usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CLUSTER>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>]

usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CONFIG>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>]

                      source_path install_path

	"

}
@@ -40,12 +50,14 @@ optional arguments: 
    -n <NAPLUGIN>, --na <NAPLUGIN>

                network layer that is used for communication. Valid: {bmi,ofi,all}

                defaults to 'all'

    -c <CLUSTER>, --cluster <CLUSTER>

                additional configurations for specific compute clusters

                supported clusters: {mogon1,mogon2,fh2}

    -c <CONFIG>, --config <CONFIG>

                allows additional configurations, e.g., for specific clusters

                supported values: {mogon2, direct, all}

                defaults to 'direct'

    -d <DEPENDENCY>, --dependency <DEPENDENCY>

                build and install a specific dependency. If unspecified 

                all dependencies are built and installed.

                download a specific dependency and ignore --config setting. If unspecified

                all dependencies are built and installed based on set --config setting.

                Multiple dependencies are supported: Pass a space-separated string (e.g., \"ofi mercury\"

    -j <COMPILE_CORES>, --compilecores <COMPILE_CORES>

                number of cores that are used to compile the dependencies

                defaults to number of available cores
@@ -53,61 +65,66 @@ optional arguments: 
"

}





list_dependencies() {



    echo "Available dependencies: "



    echo -n "  Mogon 1: "

    for d in "${MOGON1_DEPS[@]}"

    do

    echo -n "  Mogon 2: "

    for d in "${MOGON2_DEPS[@]}"; do

        echo -n "$d "

    done

    echo ""



    echo -n "  Mogon 2: "

    for d in "${MOGON2_DEPS[@]}"

    do

    echo -n "  Direct GekkoFS dependencies: "

    for d in "${DIRECT_DEPS[@]}"; do

        echo -n "$d "

    done

    echo ""



    echo -n "  fh2: "

    for d in "${FH2_DEPS[@]}"

    do

    echo -n "  All: "

    for d in "${ALL_DEPS[@]}"; do

        echo -n "$d "

    done

    echo ""

}



check_dependency() {

  local DEP=$1

  shift

  local DEP_CONFIG=("$@")

  # ignore template when specific dependency is set

  if [[ -n "${DEPENDENCY}" ]]; then

      # check if specific dependency was set and return from function

      if echo "${DEPENDENCY}" | grep -q "${DEP}"; then

        return

      fi

#      [[ "${DEPENDENCY}" == "${DEP}" ]] && return

  else

      # if not check if dependency is part of dependency config

      for e in "${DEP_CONFIG[@]}"; do

        if [[ "${DEP}" == "${e}" ]]; then

          return

        fi

      done

  fi

  false

}



prepare_build_dir() {

    if [ ! -d "$1/build" ]; then

        mkdir $1/build

        mkdir "$1"/build

    fi

    rm -rf $1/build/*

    rm -rf "$1"/build/*

}



find_cmake() {

    local CMAKE=`command -v cmake3 || command -v cmake`

    local CMAKE

    CMAKE=$(command -v cmake3 || command -v cmake)

    if [ $? -ne 0 ]; then

        >&2 echo "ERROR: could not find cmake"

        echo >&2 "ERROR: could not find cmake"

        exit 1

    fi

    echo ${CMAKE}

    echo "${CMAKE}"

}



PATCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

PATCH_DIR="${PATCH_DIR}/patches"

CLUSTER=""

DEPENDENCY=""

NA_LAYER=""

CORES=""

SOURCE=""

INSTALL=""



POSITIONAL=()

while [[ $# -gt 0 ]]

do

while [[ $# -gt 0 ]]; do

    key="$1"



    case ${key} in
@@ -116,14 +133,22 @@ case ${key} in 
        shift # past argument

        shift # past value

        ;;

	-c|--cluster)

    CLUSTER="$2"

    -c | --config)

        if [[ -z "$2" ]]; then

            echo "ERROR: Missing argument for -c/--config option"

            exit 1

        fi

        if ! echo "$VALID_DEP_OPTIONS" | grep -q "$2"; then

            echo "ERROR: Invalid argument for -c/--config option"

            exit 1

        fi

        TMP_DEP_CONF="$2"

        shift # past argument

        shift # past value

        ;;

    -d | --dependency)

        if [[ -z "$2" ]]; then

        echo "Missing argument for -d/--dependency option"

            echo "ERROR: Missing argument for -d/--dependency option"

            exit

        fi

        DEPENDENCY="$2"
@@ -158,7 +183,7 @@ set -- "${POSITIONAL[@]}" # restore positional parameters 


# deal with positional arguments

if [[ (-z ${1+x}) || (-z ${2+x}) ]]; then

    echo "Positional arguments missing."

    echo "ERROR: Positional arguments missing."

    usage_short

    exit 1

fi
@@ -166,16 +191,16 @@ SOURCE="$( readlink -mn "${1}" )" 
INSTALL="$(readlink -mn "${2}")"



# deal with optional arguments

if [ "${NA_LAYER}" == "" ]; then

	echo "Defaulting NAPLUGIN to 'all'"

	NA_LAYER="all"

if [[ "${NA_LAYER}" == "" ]]; then

    echo "Defaulting NAPLUGIN to 'ofi'"

    NA_LAYER="ofi"

fi

if [ "${CORES}" == "" ]; then

if [[ "${CORES}" == "" ]]; then

    CORES=$(grep -c ^processor /proc/cpuinfo)

    echo "CORES = ${CORES} (default)"

else

	if [ ! "${CORES}" -gt "0" ]; then

		echo "CORES set to ${CORES} which is invalid.

    if [[ ! "${CORES}" -gt "0" ]]; then

        echo "ERROR: CORES set to ${CORES} which is invalid.

Input must be numeric and greater than 0."

        usage_short

        exit
@@ -183,35 +208,40 @@ Input must be numeric and greater than 0." 
        echo CORES = "${CORES}"

    fi

fi

if [ "${NA_LAYER}" == "bmi" ] || [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then

if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then

    echo NAPLUGIN = "${NA_LAYER}"

else

    echo "No valid plugin selected"

    echo "ERROR: No valid NA plugin selected"

    usage_short

    exit

fi

if [[ "${CLUSTER}" != "" ]]; then

	if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) || ( "${CLUSTER}" == "mogon2" ) ]]; then

		echo CLUSTER  = "${CLUSTER}"

    else

        echo "${CLUSTER} cluster configuration is invalid. Exiting ..."

        usage_short

        exit

    fi

else

    echo "No cluster configuration set."

fi

# enable predefined dependency template

case ${TMP_DEP_CONF} in

mogon2)

  DEP_CONFIG=("${MOGON2_DEPS[@]}")

  echo "'Mogon2' dependencies are compiled"

  ;;

all)

  DEP_CONFIG=("${ALL_DEPS[@]}")

  echo "'All' dependencies are compiled"

  ;;

direct | *)

  DEP_CONFIG=("${DIRECT_DEPS[@]}")

  echo "'Direct' GekkoFS dependencies are compiled (default)"

  ;;

esac



USE_BMI="-DNA_USE_BMI:BOOL=OFF"

USE_OFI="-DNA_USE_OFI:BOOL=OFF"



CMAKE=`find_cmake`

CMAKE=$(find_cmake)

CMAKE="${CMAKE} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}"



echo "Source path = ${SOURCE}";

echo "Install path = ${INSTALL}";

echo "Source path = ${SOURCE}"

echo "Install path = ${INSTALL}"

echo "------------------------------------"



mkdir -p ${SOURCE}

mkdir -p "${SOURCE}"



######### From now on exits on any error ########

set -e
@@ -219,64 +249,72 @@ set -e 
export CPATH="${CPATH}:${INSTALL}/include"

export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL}/lib:${INSTALL}/lib64"



## Third party dependencies



# Set cluster dependencies first

if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) || ( "${CLUSTER}" == "mogon2" ) ]]; then



    # compile zstd

    if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "zstd" ) ]]; then

# build zstd for fast compression in rocksdb

if check_dependency "zstd" "${DEP_CONFIG[@]}"; then

    echo "############################################################ Installing:  zstd"

    CURR=${SOURCE}/zstd/build/cmake

        prepare_build_dir ${CURR}

        cd ${CURR}/build

        $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Release ..

        make -j${CORES}

    prepare_build_dir "${CURR}"

    cd "${CURR}"/build

    $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Release ..

    make -j"${CORES}"

    make install

fi



    # build lz4

    if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "zstd" ) ]]; then

        echo "############################################################ Installing:  zstd"

# build zlib for rocksdb

if check_dependency "lz4" "${DEP_CONFIG[@]}"; then

    echo "############################################################ Installing:  lz4"

    CURR=${SOURCE}/lz4

        cd ${CURR}

        make -j${CORES}

        make DESTDIR=${INSTALL} PREFIX="" install

    cd "${CURR}"

    make -j"${CORES}"

    make DESTDIR="${INSTALL}" PREFIX="" install

fi



    # build snappy

    if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "snappy" ) ]]; then

# build snappy for rocksdb

if check_dependency "snappy" "${DEP_CONFIG[@]}"; then

    echo "############################################################ Installing:  snappy"

    CURR=${SOURCE}/snappy

        prepare_build_dir ${CURR}

        cd ${CURR}/build

        $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Release ..

        make -j${CORES}

    prepare_build_dir "${CURR}"

    cd "${CURR}"/build

    $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Release ..

    make -j"${CORES}"

    make install

fi



# build capstone for syscall-intercept

if check_dependency "capstone" "${DEP_CONFIG[@]}"; then

    echo "############################################################ Installing:  capstone"

    CURR=${SOURCE}/capstone

    prepare_build_dir "${CURR}"

    cd "${CURR}"/build

    $CMAKE -DCMAKE_INSTALL_PREFIX=/home/vef/gekkofs_deps/install -DCMAKE_BUILD_TYPE:STRING=Release ..

    make -j"${CORES}" install

fi



# build bmi

if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "bmi" ) ]]; then

    if [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "all" ]; then

# build BMI

if check_dependency "bmi" "${DEP_CONFIG[@]}"; then

    if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "all" ]]; then

        USE_BMI="-DNA_USE_BMI:BOOL=ON"

        echo "############################################################ Installing:  BMI"

        # BMI

        CURR=${SOURCE}/bmi

        prepare_build_dir ${CURR}

        cd ${CURR}

        prepare_build_dir "${CURR}"

        cd "${CURR}"

        ./prepare

        cd ${CURR}/build

        CFLAGS="${CFLAGS} -w" ../configure --prefix=${INSTALL} --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict

        make -j${CORES}

        cd "${CURR}"/build

        CFLAGS="${CFLAGS} -w" ../configure --prefix="${INSTALL}" --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict

        make -j"${CORES}"

        make install

    fi

fi



# build ofi

if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "ofi" ) ]]; then

    if [ "$NA_LAYER" == "ofi" ] || [ "$NA_LAYER" == "all" ]; then

if check_dependency "ofi" "${DEP_CONFIG[@]}"; then

    if [[ "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then

        USE_OFI="-DNA_USE_OFI:BOOL=ON"

        # Mogon2 already has libfabric installed in a version that Mercury supports.

        if [[ ("${CLUSTER}" != "mogon2") ]]; then

        echo "############################################################ Installing:  LibFabric"

        #libfabric

        CURR=${SOURCE}/libfabric
@@ -288,24 +326,22 @@ if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "ofi" ) ]]; then 
        [ "${PERFORM_TEST}" ] && make check

    fi

fi

fi





# Mercury

if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "mercury" ) ]]; then

if check_dependency "mercury" "${DEP_CONFIG[@]}"; then



    if [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "all" ]; then

    if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "all" ]]; then

        USE_BMI="-DNA_USE_BMI:BOOL=ON"

    fi



    if [ "$NA_LAYER" == "ofi" ] || [ "$NA_LAYER" == "all" ]; then

    if [[ "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then

        USE_OFI="-DNA_USE_OFI:BOOL=ON"

    fi



    echo "############################################################ Installing:  Mercury"

    CURR=${SOURCE}/mercury

    prepare_build_dir ${CURR}

    cd ${CURR}/build

    prepare_build_dir "${CURR}"

    cd "${CURR}"/build

    PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig $CMAKE \

        -DCMAKE_BUILD_TYPE:STRING=Release \

        -DBUILD_TESTING:BOOL=ON \
@@ -318,67 +354,66 @@ if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "mercury" ) ]]; then 
        -DCMAKE_INSTALL_PREFIX=${INSTALL} \

        ${USE_BMI} ${USE_OFI} \

        ..

    make -j${CORES}

    make -j"${CORES}"

    make install

fi



# Argobots

if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "argobots" ) ]]; then

if check_dependency "argobots" "${DEP_CONFIG[@]}"; then

    echo "############################################################ Installing:  Argobots"

    CURR=${SOURCE}/argobots

    prepare_build_dir ${CURR}

    cd ${CURR}

    prepare_build_dir "${CURR}"

    cd "${CURR}"

    ./autogen.sh

    cd ${CURR}/build

    ../configure --prefix=${INSTALL} --enable-perf-opt --disable-checks

    make -j${CORES}

    cd "${CURR}"/build

    ../configure --prefix="${INSTALL}" --enable-perf-opt --disable-checks

    make -j"${CORES}"

    make install

    [ "${PERFORM_TEST}" ] && make check

fi



# Margo

if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "margo" ) ]]; then

if check_dependency "margo" "${DEP_CONFIG[@]}"; then

    echo "############################################################ Installing:  Margo"

    CURR=${SOURCE}/margo

    prepare_build_dir ${CURR}

    cd ${CURR}

    prepare_build_dir "${CURR}"

    cd "${CURR}"

    ./prepare.sh

    cd ${CURR}/build

    ../configure --prefix=${INSTALL} PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig CFLAGS="${CFLAGS} -Wall -O3"

    make -j${CORES}

    cd "${CURR}"/build

    ../configure --prefix="${INSTALL}" PKG_CONFIG_PATH="${INSTALL}"/lib/pkgconfig CFLAGS="${CFLAGS} -Wall -O3"

    make -j"${CORES}"

    make install

    [ "${PERFORM_TEST}" ] && make check

fi



# Rocksdb

if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "rocksdb" ) ]]; then

if check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then

    echo "############################################################ Installing:  Rocksdb"

    CURR=${SOURCE}/rocksdb

    cd ${CURR}

    cd "${CURR}"

    make clean

    USE_RTTI=1 make -j${CORES} static_lib

    INSTALL_PATH=${INSTALL} make install

    USE_RTTI=1 make -j"${CORES}" static_lib

    INSTALL_PATH="${INSTALL}" make install

fi



# syscall_intercept

if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "syscall_intercept" ) ]]; then

if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then

    echo "############################################################ Installing:  Syscall_intercept"

    CURR=${SOURCE}/syscall_intercept

    prepare_build_dir ${CURR}

    cd ${CURR}/build

    $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF ..

    prepare_build_dir "${CURR}"

    cd "${CURR}"/build

    $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF ..

    make install

fi



# date

if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "date" ) ]]; then

if check_dependency "date" "${DEP_CONFIG[@]}"; then

    echo "############################################################ Installing:  date"

    CURR=${SOURCE}/date

    prepare_build_dir ${CURR}

    cd ${CURR}/build

    $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON ..

    prepare_build_dir "${CURR}"

    cd "${CURR}"/build

    $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON ..

    make install

fi





echo "Done"

scripts/dl_dep.sh

+131 −90

File changed.

Preview size limit exceeded, changes collapsed.