Newer
Older
local FOLDER=$1
local CLONE=$2
local COMMIT=$3
local ACTION
if [ -d "${SOURCE}/${FOLDER}" ]; then
echo "${FOLDER} directory exists. Pulling instead."
cd ${SOURCE}/${FOLDER} && git pull origin master &>> ${LOG}
else
cd ${SOURCE} && ${CLONE} &>> ${LOG}
cd ${SOURCE}/${FOLDER} && git checkout -f ${COMMIT} &>> ${LOG}
echo "${ACTION} ${FOLDER} [$COMMIT]"
}
wgetdeps() {
FOLDER=$1
URL=$2
if [ -d "${SOURCE}/${FOLDER}" ]; then
echo "${FOLDER} directory exists. Removing its content first." &>> ${LOG}
rm -rf ${SOURCE}/${FOLDER}/* &>> ${LOG}
else
mkdir ${SOURCE}/${FOLDER}
fi
cd ${SOURCE}
FILENAME=$(basename $URL)
if [ -f "${SOURCE}/$FILENAME" ]; then
rm ${SOURCE}/$FILENAME
fi
wget $URL &>> ${LOG} || exit 1
tar -xf $FILENAME --directory ${SOURCE}/${FOLDER} --strip-components=1 &>> ${LOG}
rm $FILENAME
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
}
usage_short() {
echo "
usage: dl_dep.sh [-h] [-n <NAPLUGIN>] [-c <CLUSTER>]
source_path
"
}
help_msg() {
usage_short
echo "
This script gets all ADA-FS dependency sources (excluding the fs itself)
positional arguments:
source_path path where the dependency downloads are put
optional arguments:
-h, --help shows this help message and exits
-n <NAPLUGIN>, --na <NAPLUGIN>
network layer that is used for communication. Valid: {bmi,cci,ofi,all}
defaults to 'all'
-c <CLUSTER>, --cluster <CLUSTER>
additional configurations for specific compute clusters
supported clusters: {mogon1,fh2}
"
}
CLUSTER=""
NA_LAYER=""
POSITIONAL=()
while [[ $# -gt 0 ]]
do
key="$1"
case ${key} in
-n|--na)
NA_LAYER="$2"
shift # past argument
shift # past value
;;
-c|--cluster)
CLUSTER="$2"
shift # past argument
shift # past value
;;
-h|--help)
help_msg
exit
#shift # past argument
;;
*) # unknown option
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
set -- "${POSITIONAL[@]}" # restore positional parameters
# positional arguments
if [[ -z ${1+x} ]]; then
echo "Positional arguments missing."
usage_short
exit
fi
SOURCE=$1
LOG="/tmp/adafs_download_deps.log"
echo "" &> ${LOG}
# optional arguments
if [ "${NA_LAYER}" == "" ]; then
echo "Defaulting NAPLUGIN to 'all'"
NA_LAYER="all"
fi
# sanity checks
if [[ ( "${NA_LAYER}" == "cci" ) || ( "${NA_LAYER}" == "bmi" ) || ( "${NA_LAYER}" == "ofi" ) || ( "${NA_LAYER}" == "all" ) ]]; then
echo NAPLUGIN = "${NA_LAYER}"
else
echo "No valid plugin selected"
usage_short
exit
fi
if [[ "${CLUSTER}" != "" ]]; then
if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) ]]; then
echo CLUSTER = "${CLUSTER}"
else
echo "${CLUSTER} cluster configuration is invalid. Exiting ..."
usage_short
exit
fi
else
echo "No cluster configuration set."
fi
echo "Source path is set to \"${SOURCE}\""
echo "Download progress is logged at \"${LOG}\""
mkdir -p ${SOURCE}
# get cluster dependencies
if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) ]]; then
# get libtool for cci
wgetdeps "libtool" "https://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz" &
wgetdeps "libev" "http://dist.schmorp.de/libev/libev-4.24.tar.gz" &
wgetdeps "gflags" "https://github.com/gflags/gflags/archive/v2.2.1.tar.gz" &
# get zstd for fast compression in rocksdb
wgetdeps "zstd" "https://github.com/facebook/zstd/archive/v1.3.2.tar.gz" &
wgetdeps "lz4" "https://github.com/lz4/lz4/archive/v1.8.0.tar.gz" &
wgetdeps "snappy" "https://github.com/google/snappy/archive/1.1.7.tar.gz" &
fi
#if [ "${CLUSTER}" == "fh2" ]; then
# no distinct 3rd party software needed as of now.
#fi
# get BMI
if [ "${NA_LAYER}" == "bmi" ] || [ "${NA_LAYER}" == "all" ]; then
clonedeps "bmi" "git clone git://git.mcs.anl.gov/bmi" "2abbe991edc45b713e64c5fed78a20fdaddae59b" &
fi
# get CCI
if [ "${NA_LAYER}" == "cci" ] || [ "${NA_LAYER}" == "all" ]; then
clonedeps "cci" "git clone https://github.com/CCI/cci" "58fd58ea2aa60c116c2b77c5653ae36d854d78f2" &
fi
# get libfabric
if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then
wgetdeps "libfabric" "https://github.com/ofiwg/libfabric/archive/v1.5.3.tar.gz" &
clonedeps "mercury" "git clone --recurse-submodules https://github.com/mercury-hpc/mercury" "c4faa382fd228c0b629c9164a984df1779089d3f" &
clonedeps "argobots" "git clone -b dev-get-dev-basic https://github.com/carns/argobots.git" "78ceea28ed44faca12cf8ea7f5687b894c66a8c4" &
clonedeps "abt-snoozer" "git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git" "3d9240eda290bfb89f08a5673cebd888194a4bd7" &
# get Argobots-IO
#clonedeps "abt-io" "git clone https://xgitlab.cels.anl.gov/sds/abt-io.git" "35f16da88a1c579ed4726bfa77daa1884829fc0c" &
clonedeps "margo" "git clone https://xgitlab.cels.anl.gov/sds/margo.git" "72eec057314a4251d8658e03a18240275992e1ce" &
wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v5.10.3.tar.gz" &
# Wait for all download to be completed
wait