Commit db3ed504 authored by Marc Vef's avatar Marc Vef
Browse files

Added mogon1 configurations and dependencies download+compilation to dep scripts

parent a8d91a6b

ifs/scripts/clone_dep.sh

+56 −11
Original line number Diff line number Diff line 
#!/bin/bash



assertdir()

{

clonedeps() {

    FOLDER=$1

    GITCLONE=$2

    COMMIT=$3
@@ -21,11 +20,30 @@ assertdir() 
    echo "Done"

}



wgetdeps() {

    FOLDER=$1

    URL=$2

    echo "#########################################################"

    echo "Wgetting into $GIT/$FOLDER ..."

    if [ -d "$GIT/$FOLDER" ]; then

        echo "$FOLDER directory exists. Removing its content first."

        rm -rf $GIT/$FOLDER/* &>> $LOG

    else

        mkdir $GIT/$FOLDER

    fi

    cd $GIT

    wget $URL &>> $LOG || exit 1

    FILENAME =$(basename http://ftpmirror.gnu.org/libtool/libtool-2.4.6.tar.gz)

    tar -xf $FILENAME --directory $GIT/$FOLDER  &>> $LOG

    rm $FILENAME

}



usage() {



    echo "Usage:

    ./clone_dep [ clone_path ] [ NA_Plugin ]

    Valid NA_Plugin arguments: {bmi,cci,ofi,all}"

    Valid NA_Plugin arguments: {bmi,cci,ofi,all}

    Valid cluster arguments: {mogon1}"

}



if [[ ( -z ${1+x} ) || ( -z ${2+x} ) ]]; then
@@ -33,11 +51,17 @@ if [[ ( -z ${1+x} ) || ( -z ${2+x} ) ]]; then 
    usage

    exit

fi

# if cluster is given, put it into a variable

CLUSTER=""

if [[ (-z ${3+x} ) ]]; then

    CLUSTER=$3

fi



LOG=/tmp/adafs_clone.log

echo "" &> $LOG

GIT=$1

NA_LAYER=$2

# sanity checks

if [ "$NA_LAYER" == "cci" ] || [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "ofi" ] || [ "$NA_LAYER" == "all" ]; then

    echo "$NA_LAYER plugin(s) selected"

else
@@ -45,32 +69,53 @@ else 
    usage

    exit

fi

if [ "$CLUSTER" != "" ]; then

    if [ "$CLUSTER" == "mogon1" ]; then

        echo "$CLUSTER cluster configuration selected"

    else

        echo "$CLUSTER cluster configuration is invalid. Exiting ..."

        usage

        exit

    fi

else

    echo "No cluster configuration set."

fi



echo "Clone path is set to '$1'"

echo "Cloning output is logged at /tmp/adafs_clone.log"



mkdir -p $GIT



# get cluster dependencies

if [ "$CLUSTER" == "mogon1" ]; then

    # get libtool for cci

    wgetdeps "libtool" "http://ftpmirror.gnu.org/libtool/libtool-2.4.6.tar.gz"

    # get libev for mercury

    wgetdeps "libev" "http://dist.schmorp.de/libev/libev-4.24.tar.gz"

    # TODO rocksdb deps

fi



# get BMI

if [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "all" ]; then

    assertdir "bmi" "git clone git://git.mcs.anl.gov/bmi" "2abbe991edc45b713e64c5fed78a20fdaddae59b"

    clonedeps "bmi" "git clone git://git.mcs.anl.gov/bmi" "2abbe991edc45b713e64c5fed78a20fdaddae59b"

fi

# get CCI

if [ "$NA_LAYER" == "cci" ] || [ "$NA_LAYER" == "all" ]; then

    assertdir "cci" "git clone https://github.com/CCI/cci" "58fd58ea2aa60c116c2b77c5653ae36d854d78f2"

    clonedeps "cci" "git clone https://github.com/CCI/cci" "58fd58ea2aa60c116c2b77c5653ae36d854d78f2"

fi

# get libfabric

if [ "$NA_LAYER" == "ofi" ] || [ "$NA_LAYER" == "all" ]; then

    assertdir "libfabric" "git clone https://github.com/ofiwg/libfabric" "tags/v1.5.2"

    clonedeps "libfabric" "git clone https://github.com/ofiwg/libfabric" "tags/v1.5.2"

fi

# get Mercury

assertdir "mercury" "git clone --recurse-submodules https://github.com/mercury-hpc/mercury" "afd70055d21a6df2faefe38d5f6ce1ae11f365a5"

clonedeps "mercury" "git clone --recurse-submodules https://github.com/mercury-hpc/mercury" "afd70055d21a6df2faefe38d5f6ce1ae11f365a5"

# get Argobots

assertdir "argobots" "git clone -b dev-get-dev-basic https://github.com/carns/argobots.git" "a5a6b2036c75ad05804ccb72d2fe31cea1bfef88"

clonedeps "argobots" "git clone -b dev-get-dev-basic https://github.com/carns/argobots.git" "a5a6b2036c75ad05804ccb72d2fe31cea1bfef88"

# get Argobots-snoozer

assertdir "abt-snoozer" "git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git" "3d9240eda290bfb89f08a5673cebd888194a4bd7"

clonedeps "abt-snoozer" "git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git" "3d9240eda290bfb89f08a5673cebd888194a4bd7"

# get Margo

assertdir "margo" "git clone https://xgitlab.cels.anl.gov/sds/margo.git" "68ef7f14178e9066cf38846d90d451e00aaca61d"

clonedeps "margo" "git clone https://xgitlab.cels.anl.gov/sds/margo.git" "68ef7f14178e9066cf38846d90d451e00aaca61d"

# get rocksdb

assertdir "rocksdb" "git clone https://github.com/facebook/rocksdb" "tags/v5.8"

clonedeps "rocksdb" "git clone https://github.com/facebook/rocksdb" "tags/v5.8"



echo "Nothing left to do. Exiting."

ifs/scripts/compile_dep.sh

+45 −2
Original line number Diff line number Diff line
@@ -3,8 +3,9 @@ 
usage() {



    echo "Usage:

    ./compile_dep [ clone_path ] [ install_path ] [ na_plugin ]

    Valid na_plugin arguments: {bmi,cci,ofi,all}"

    ./compile_dep [ clone_path ] [ install_path ] [ na_plugin ] [ cluster ]

    Valid na_plugin arguments: {bmi,cci,ofi,all}

    Valid cluster arguments: {mogon1}"

}



prepare_build_dir() {
@@ -19,6 +20,11 @@ if [[ ( -z ${1+x} ) || ( -z ${2+x} ) || ( -z ${3+x} ) ]]; then 
    usage

    exit

fi

# if cluster is given, put it into a variable

CLUSTER=""

if [[ (-z ${4+x} ) ]]; then

    CLUSTER=$4

fi



#LOG=/tmp/adafs_install.log

#echo "" &> $LOG
@@ -39,11 +45,48 @@ else 
    exit

fi



if [ "$CLUSTER" != "" ]; then

    if [ "$CLUSTER" == "mogon1" ]; then

        echo "$CLUSTER cluster configuration selected"

    else

        echo "$CLUSTER cluster configuration is invalid. Exiting ..."

        usage

        exit

    fi

else

    echo "No cluster configuration set."

fi







echo "Git path is set to '$1'";

echo "Install path is set to '$2'";



mkdir -p $GIT



# Set cluster dependencies first

if [ "$CLUSTER" == "mogon1" ]; then

    # load required modules

    module load devel/CMake/3.8.0 || exit 1

    module load mpi/MVAPICH2/2.2-GCC-6.3.0-slurm || exit 1

    module load devel/Boost/1.63.0-intel-2017.02-Python-2.7.13 || exit 1 # because of mercury

    # get libtool

    CURR=$GIT/libtool

    prepare_build_dir $CURR

    cd $CURR/build

    ../configure --prefix=$INSTALL || exit 1

    make -j$CORES || exit 1

    make install || exit 1

    # compile libev

    CURR=$GIT/libev

    prepare_build_dir $CURR

    cd $CURR/build

    ../configure --prefix=$INSTALL || exit 1

    make -j$CORES || exit 1

    make install || exit 1

    # TODO rocksdb deps

fi



if [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "all" ]; then

    USE_BMI="-DNA_USE_BMI:BOOL=ON"

    echo "Installing BMI"