Commit 615e634b authored by Marc Vef's avatar Marc Vef
Browse files

Better argument parsing for scripts etc

parent 61690569

ifs/scripts/compile_dep.sh

+25 −20
Original line number Diff line number Diff line
@@ -3,7 +3,7 @@ 
usage_short() {

	echo "

usage: compile_dep.sh [-h] [-n <NAPLUGIN>] [-c <CLUSTER>] [-j <COMPILE_CORES>]

                      clone_path install_path

                      source_path install_path

	"

}
@@ -14,7 +14,7 @@ help_msg() { 
This script compiles all ADA-FS dependencies (excluding the fs itself)



positional arguments:

	clone_path		path to the cloned dependencies path from clone_dep.sh

	source_path		path to the cloned dependencies path from clone_dep.sh

	install_path		path to the install path of the compiled dependencies
@@ -38,6 +38,11 @@ prepare_build_dir() { 
    fi

    rm -rf $1/build/*

}

CLUSTER=""

NA_LAYER=""

CORES=""

SOURCE=""

INSTALL=""



POSITIONAL=()

while [[ $# -gt 0 ]]
@@ -79,7 +84,7 @@ if [[ ( -z ${1+x} ) || ( -z ${2+x} ) ]]; then 
    usage_short

    exit

fi

GIT=$1

SOURCE=$1

INSTALL=$2



# deal with optional arguments
@@ -107,7 +112,7 @@ else 
    usage_short

    exit

fi

if [[ -v ${CLUSTER} ]]; then

if [[ "${CLUSTER}" != "" ]]; then

	if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) ]]; then

		echo CLUSTER  = "${CLUSTER}"

    else
@@ -125,16 +130,16 @@ USE_BMI="-DNA_USE_BMI:BOOL=OFF" 
USE_CCI="-DNA_USE_CCI:BOOL=OFF"

USE_OFI="-DNA_USE_OFI:BOOL=OFF"



echo "Git path = '$1'";

echo "Source path = '$1'";

echo "Install path = '$2'";



mkdir -p ${GIT}

mkdir -p ${SOURCE}



# Set cluster dependencies first

if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) ]]; then

    # get libtool

    echo "############################################################ Installing:  libtool"

    CURR=${GIT}/libtool

    CURR=${SOURCE}/libtool

    prepare_build_dir ${CURR}

    cd ${CURR}/build

    ../configure --prefix=${INSTALL} || exit 1
@@ -142,7 +147,7 @@ if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) ]]; then 
    make install || exit 1

    # compile libev

    echo "############################################################ Installing:  libev"

    CURR=${GIT}/libev

    CURR=${SOURCE}/libev

    prepare_build_dir ${CURR}

    cd ${CURR}/build

    ../configure --prefix=${INSTALL} || exit 1
@@ -150,7 +155,7 @@ if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) ]]; then 
    make install || exit 1

    # compile gflags

    echo "############################################################ Installing:  gflags"

    CURR=${GIT}/gflags

    CURR=${SOURCE}/gflags

    prepare_build_dir ${CURR}

    cd ${CURR}/build

    cmake -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Release .. || exit 1
@@ -158,19 +163,19 @@ if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) ]]; then 
    make install || exit 1

    # compile zstd

    echo "############################################################ Installing:  zstd"

    CURR=${GIT}/zstd/build/cmake

    CURR=${SOURCE}/zstd/build/cmake

    prepare_build_dir ${CURR}

    cd ${CURR}/build

    cmake -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Release .. || exit 1

    make -j${CORES} || exit 1

    make install || exit 1

    echo "############################################################ Installing:  lz4"

    CURR=${GIT}/lz4

    CURR=${SOURCE}/lz4

	cd ${CURR}

    make -j${CORES} || exit 1

    make DESTDIR=${INSTALL} PREFIX="" install || exit 1

    echo "############################################################ Installing:  snappy"

    CURR=${GIT}/snappy

    CURR=${SOURCE}/snappy

    prepare_build_dir ${CURR}

    cd ${CURR}/build

    cmake -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Release .. || exit 1
@@ -182,7 +187,7 @@ if [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "all" ]; then 
    USE_BMI="-DNA_USE_BMI:BOOL=ON"

    echo "############################################################ Installing:  BMI"

    # BMI

    CURR=${GIT}/bmi

    CURR=${SOURCE}/bmi

    prepare_build_dir ${CURR}

    cd ${CURR}

    ./prepare || exit 1
@@ -196,7 +201,7 @@ if [ "$NA_LAYER" == "cci" ] || [ "$NA_LAYER" == "all" ]; then 
    USE_CCI="-DNA_USE_CCI:BOOL=ON"

    echo "############################################################ Installing:  CCI"

    # CCI

    CURR=${GIT}/cci

    CURR=${SOURCE}/cci

    prepare_build_dir ${CURR}

    cd ${CURR}

    ./autogen.pl || exit 1
@@ -215,7 +220,7 @@ if [ "$NA_LAYER" == "ofi" ] || [ "$NA_LAYER" == "all" ]; then 
    USE_OFI="-DNA_USE_OFI:BOOL=ON"

    echo "############################################################ Installing:  LibFabric"

    #libfabric

    CURR=${GIT}/libfabric

    CURR=${SOURCE}/libfabric

    prepare_build_dir ${CURR}

    cd ${CURR}

    ./autogen.sh || exit 1
@@ -229,7 +234,7 @@ fi 
echo "############################################################ Installing:  Mercury"



# Mercury

CURR=${GIT}/mercury

CURR=${SOURCE}/mercury

prepare_build_dir ${CURR}

cd ${CURR}/build

# XXX Note: USE_EAGER_BULK is temporarily disabled due to bugs in Mercury with smaller amounts of data
@@ -242,7 +247,7 @@ make install || exit 1 
echo "############################################################ Installing:  Argobots"



# Argobots

CURR=${GIT}/argobots

CURR=${SOURCE}/argobots

prepare_build_dir ${CURR}

cd ${CURR}

./autogen.sh || exit 1
@@ -254,7 +259,7 @@ make check || exit 1 


echo "############################################################ Installing:  Abt-snoozer"

# Abt snoozer

CURR=${GIT}/abt-snoozer

CURR=${SOURCE}/abt-snoozer

prepare_build_dir ${CURR}

cd ${CURR}

./prepare.sh || exit 1
@@ -266,7 +271,7 @@ make check || exit 1 


echo "############################################################ Installing:  Margo"

# Margo

CURR=${GIT}/margo

CURR=${SOURCE}/margo

prepare_build_dir ${CURR}

cd ${CURR}

./prepare.sh || exit 1
@@ -278,7 +283,7 @@ make check || exit 1 


echo "############################################################ Installing:  Rocksdb"

# Rocksdb

CURR=${GIT}/rocksdb

CURR=${SOURCE}/rocksdb

cd ${CURR}

make clean || exit 1

sed -i.bak "s#INSTALL_PATH ?= /usr/local#INSTALL_PATH ?= ${INSTALL}#g" Makefile

ifs/scripts/clone_dep.sh→ifs/scripts/dl_dep.sh

+187 −0
Original line number Diff line number Diff line
@@ -2,21 +2,21 @@ 


clonedeps() {

    FOLDER=$1

    GITCLONE=$2

    CLONE=$2

    COMMIT=$3



    echo "#########################################################"

    echo "Cloning into $GIT/$FOLDER ..."

    echo "Cloning into ${SOURCE}/${FOLDER} ..."





    if [ -d "$GIT/$FOLDER" ]; then

        echo "$FOLDER directory exists. Pulling instead."

        cd $GIT/$FOLDER && git pull origin master &>> $LOG

    if [ -d "${SOURCE}/${FOLDER}" ]; then

        echo "${FOLDER} directory exists. Pulling instead."

        cd ${SOURCE}/${FOLDER} && git pull origin master &>> ${LOG}

    else

        cd $GIT && $GITCLONE &>> $LOG

        cd ${SOURCE} && ${CLONE} &>> ${LOG}

    fi

    # fix the version

    cd $GIT/$FOLDER && git checkout $COMMIT &>> $LOG

    cd ${SOURCE}/${FOLDER} && git checkout ${COMMIT} &>> ${LOG}

    echo "Done"

}
@@ -24,74 +24,126 @@ wgetdeps() { 
    FOLDER=$1

    URL=$2

    echo "#########################################################"

    echo "Wgetting into $GIT/$FOLDER ..."

    if [ -d "$GIT/$FOLDER" ]; then

        echo "$FOLDER directory exists. Removing its content first."

        rm -rf $GIT/$FOLDER/* &>> $LOG

    echo "Wgetting into ${SOURCE}/${FOLDER} ..."

    if [ -d "${SOURCE}/${FOLDER}" ]; then

        echo "${FOLDER} directory exists. Removing its content first."

        rm -rf ${SOURCE}/${FOLDER}/* &>> ${LOG}

    else

        mkdir $GIT/$FOLDER

        mkdir ${SOURCE}/${FOLDER}

    fi

    cd $GIT

    cd ${SOURCE}

    FILENAME=$(basename $URL)

    if [ -f "$GIT/$FILENAME" ]; then

        rm $GIT/$FILENAME

    if [ -f "${SOURCE}/$FILENAME" ]; then

        rm ${SOURCE}/$FILENAME

    fi

    wget $URL &>> $LOG || exit 1

    tar -xf $FILENAME --directory $GIT/$FOLDER --strip-components=1 &>> $LOG

    wget $URL &>> ${LOG} || exit 1

    tar -xf $FILENAME --directory ${SOURCE}/${FOLDER} --strip-components=1 &>> ${LOG}

    rm $FILENAME

    echo "Done"

}



usage() {

usage_short() {

	echo "

usage: dl_dep.sh [-h] [-n <NAPLUGIN>] [-c <CLUSTER>]

                    source_path

	"

}



help_msg() {



        usage_short

    echo "

This script gets all ADA-FS dependency sources (excluding the fs itself)



    echo "Usage:

    ./clone_dep [ clone_path ] [ NA_Plugin ]

    Valid NA_Plugin arguments: {bmi,cci,ofi,all}

    Valid cluster arguments: {mogon1,fh2}"

positional arguments:

        source_path              path where the dependency downloads are put





optional arguments:

        -h, --help              shows this help message and exits

        -n <NAPLUGIN>, --na <NAPLUGIN>

                                network layer that is used for communication. Valid: {bmi,cci,ofi,all}

                                defaults to 'all'

        -c <CLUSTER>, --cluster <CLUSTER>

                                additional configurations for specific compute clusters

                                supported clusters: {mogon1,fh2}

        "

}

CLUSTER=""

NA_LAYER=""



if [[ ( -z ${1+x} ) || ( -z ${2+x} ) ]]; then

    echo "Arguments missing."

    usage

POSITIONAL=()

while [[ $# -gt 0 ]]

do

key="$1"



case ${key} in

    -n|--na)

    NA_LAYER="$2"

    shift # past argument

    shift # past value

    ;;

    -c|--cluster)

    CLUSTER="$2"

    shift # past argument

    shift # past value

    ;;

    -h|--help)

    help_msg

    exit

    #shift # past argument

    ;;

    *)    # unknown option

    POSITIONAL+=("$1") # save it in an array for later

    shift # past argument

    ;;

esac

done

set -- "${POSITIONAL[@]}" # restore positional parameters



# positional arguments

if [[ -z ${1+x} ]]; then

    echo "Positional arguments missing."

    usage_short

    exit

fi

# if cluster is given, put it into a variable

CLUSTER=""

if [[ ! (-z ${3+x} ) ]]; then

    CLUSTER=$3

SOURCE=$1



LOG="/tmp/adafs_download_deps.log"

echo "" &> ${LOG}

# optional arguments

if [ "${NA_LAYER}" == "" ]; then

        echo "Defaulting NAPLUGIN to 'all'"

        NA_LAYER="all"

fi



LOG=/tmp/adafs_clone.log

echo "" &> $LOG

GIT=$1

NA_LAYER=$2

# sanity checks

if [ "$NA_LAYER" == "cci" ] || [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "ofi" ] || [ "$NA_LAYER" == "all" ]; then

    echo "$NA_LAYER plugin(s) selected"

if [[ ( "${NA_LAYER}" == "cci" ) || ( "${NA_LAYER}" == "bmi" ) || ( "${NA_LAYER}" == "ofi" ) || ( "${NA_LAYER}" == "all" ) ]]; then

	echo NAPLUGIN = "${NA_LAYER}"

else

    echo "No valid plugin selected"

    usage

    usage_short

    exit

fi

if [ "$CLUSTER" != "" ]; then

	if [[ ( "$CLUSTER" == "mogon1" ) || ( "$CLUSTER" == "fh2" ) ]]; then

        echo "$CLUSTER cluster configuration selected"

if [[ "${CLUSTER}" != "" ]]; then

	if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) ]]; then

		echo CLUSTER  = "${CLUSTER}"

    else

        echo "$CLUSTER cluster configuration is invalid. Exiting ..."

        usage

        echo "${CLUSTER} cluster configuration is invalid. Exiting ..."

        usage_short

        exit

    fi

else

    echo "No cluster configuration set."

fi



echo "Clone path is set to '$1'"

echo "Cloning output is logged at /tmp/adafs_clone.log"

echo "Source path is set to '$1'"

echo "Download progress is logged at /tmp/adafs_download_deps.log"



mkdir -p $GIT

mkdir -p ${SOURCE}



# get cluster dependencies

if [[ ( "$CLUSTER" == "mogon1" ) || ( "$CLUSTER" == "fh2" ) ]]; then

if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) ]]; then

    # get libtool for cci

    wgetdeps "libtool" "http://ftpmirror.gnu.org/libtool/libtool-2.4.6.tar.gz"

    # get libev for mercury
@@ -105,20 +157,20 @@ if [[ ( "$CLUSTER" == "mogon1" ) || ( "$CLUSTER" == "fh2" ) ]]; then 
	# get snappy for rocksdb

    wgetdeps "snappy" "https://github.com/google/snappy/archive/1.1.7.tar.gz"

fi

#if [ "$CLUSTER" == "fh2" ]; then

#if [ "${CLUSTER}" == "fh2" ]; then

	# no distinct 3rd party software needed as of now.

#fi



# get BMI

if [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "all" ]; then

if [ "${NA_LAYER}" == "bmi" ] || [ "${NA_LAYER}" == "all" ]; then

    clonedeps "bmi" "git clone git://git.mcs.anl.gov/bmi" "2abbe991edc45b713e64c5fed78a20fdaddae59b"

fi

# get CCI

if [ "$NA_LAYER" == "cci" ] || [ "$NA_LAYER" == "all" ]; then

if [ "${NA_LAYER}" == "cci" ] || [ "${NA_LAYER}" == "all" ]; then

    clonedeps "cci" "git clone https://github.com/CCI/cci" "58fd58ea2aa60c116c2b77c5653ae36d854d78f2"

fi

# get libfabric

if [ "$NA_LAYER" == "ofi" ] || [ "$NA_LAYER" == "all" ]; then

if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then

    clonedeps "libfabric" "git clone https://github.com/ofiwg/libfabric" "tags/v1.5.2"

fi

# get Mercury