Newer
Older
#set -x
COMMON_WGET_FLAGS="--no-verbose"
COMMON_GIT_FLAGS="--quiet --single-branch"
# Stop all backround jobs on interruption.
# "kill -- -$$" sends a SIGTERM to the whole process group,
# thus killing also descendants.
trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
if [ -d "${SOURCE}/${FOLDER}/.git" ]; then
cd ${SOURCE}/${FOLDER} && git fetch -q || exit 1
git clone ${COMMON_GIT_FLAGS} ${GIT_FLAGS} -- "${REPO}" "${SOURCE}/${FOLDER}" > /dev/null || exit 1
cd "${SOURCE}/${FOLDER}" && git checkout -qf ${COMMIT} || exit 1
echo "${ACTION} ${FOLDER} [$COMMIT]"
}
wgetdeps() {
FOLDER=$1
URL=$2
if [ -d "${SOURCE}/${FOLDER}" ]; then
mkdir -p "${SOURCE}/${FOLDER}"
cd ${SOURCE}
FILENAME=$(basename $URL)
if [ -f "${SOURCE}/$FILENAME" ]; then
wget -q "$URL" || exit 1
tar -xf "$FILENAME" --directory "${SOURCE}/${FOLDER}" --strip-components=1 || exit 1
rm -f "$FILENAME"
}
usage_short() {
echo "
usage: dl_dep.sh [-h] [-n <NAPLUGIN>] [-c <CLUSTER>]
source_path
"
}
help_msg() {
usage_short
echo "
This script gets all ADA-FS dependency sources (excluding the fs itself)
positional arguments:
source_path path where the dependency downloads are put
optional arguments:
-h, --help shows this help message and exits
-n <NAPLUGIN>, --na <NAPLUGIN>
network layer that is used for communication. Valid: {bmi,cci,ofi,all}
defaults to 'all'
-c <CLUSTER>, --cluster <CLUSTER>
additional configurations for specific compute clusters
supported clusters: {mogon1,mogon2,fh2}
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
"
}
CLUSTER=""
NA_LAYER=""
POSITIONAL=()
while [[ $# -gt 0 ]]
do
key="$1"
case ${key} in
-n|--na)
NA_LAYER="$2"
shift # past argument
shift # past value
;;
-c|--cluster)
CLUSTER="$2"
shift # past argument
shift # past value
;;
-h|--help)
help_msg
exit
#shift # past argument
;;
*) # unknown option
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
set -- "${POSITIONAL[@]}" # restore positional parameters
# positional arguments
if [[ -z ${1+x} ]]; then
echo "Positional arguments missing."
usage_short
exit
fi
SOURCE="$( readlink -mn "${1}" )"
# optional arguments
if [ "${NA_LAYER}" == "" ]; then
echo "Defaulting NAPLUGIN to 'all'"
NA_LAYER="all"
fi
# sanity checks
if [[ ( "${NA_LAYER}" == "cci" ) || ( "${NA_LAYER}" == "bmi" ) || ( "${NA_LAYER}" == "ofi" ) || ( "${NA_LAYER}" == "all" ) ]]; then
echo NAPLUGIN = "${NA_LAYER}"
else
echo "No valid plugin selected"
usage_short
exit
fi
if [[ "${CLUSTER}" != "" ]]; then
if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "mogon2" ) || ( "${CLUSTER}" == "fh2" ) ]]; then
echo CLUSTER = "${CLUSTER}"
else
echo "${CLUSTER} cluster configuration is invalid. Exiting ..."
usage_short
exit
fi
else
echo "No cluster configuration set."
fi
echo "Source path is set to \"${SOURCE}\""
mkdir -p ${SOURCE}
# get cluster dependencies
if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "mogon2" ) || ( "${CLUSTER}" == "fh2" ) ]]; then
wgetdeps "libtool" "https://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz" &
wgetdeps "libev" "http://dist.schmorp.de/libev/libev-4.24.tar.gz" &
wgetdeps "gflags" "https://github.com/gflags/gflags/archive/v2.2.1.tar.gz" &
# get zstd for fast compression in rocksdb
wgetdeps "zstd" "https://github.com/facebook/zstd/archive/v1.3.2.tar.gz" &
wgetdeps "lz4" "https://github.com/lz4/lz4/archive/v1.8.0.tar.gz" &
wgetdeps "snappy" "https://github.com/google/snappy/archive/1.1.7.tar.gz" &
fi
#if [ "${CLUSTER}" == "fh2" ]; then
# no distinct 3rd party software needed as of now.
#fi
# get BMI
if [ "${NA_LAYER}" == "bmi" ] || [ "${NA_LAYER}" == "all" ]; then
clonedeps "bmi" "git://git.mcs.anl.gov/bmi" "2abbe991edc45b713e64c5fed78a20fdaddae59b" &
fi
# get CCI
if [ "${NA_LAYER}" == "cci" ] || [ "${NA_LAYER}" == "all" ]; then
clonedeps "cci" "https://github.com/CCI/cci" "58fd58ea2aa60c116c2b77c5653ae36d854d78f2" &
fi
# get libfabric
if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then
# No need to get libfabric for mogon2 as it is already installed
if [[ ("${CLUSTER}" != "mogon2") ]]; then
wgetdeps "libfabric" "https://github.com/ofiwg/libfabric/archive/v1.6.0.tar.gz" &
clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "d015745ce25d839b8b46e68c11a7d8278423a46b" "--recurse-submodules" &
clonedeps "argobots" "https://github.com/carns/argobots.git" "78ceea28ed44faca12cf8ea7f5687b894c66a8c4" "-b dev-get-dev-basic" &
clonedeps "abt-snoozer" "https://xgitlab.cels.anl.gov/sds/abt-snoozer.git" "54c506103cf2d77bd76460db29a67a875f5c5a85" &
# get Argobots-IO
#clonedeps "abt-io" "https://xgitlab.cels.anl.gov/sds/abt-io.git" "35f16da88a1c579ed4726bfa77daa1884829fc0c" &
clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "a660d149e8c9b840f9a363db03f2ef23476f91f1" &
wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v5.11.3.tar.gz" &
# Wait for all download to be completed
wait