Newer
Older
################################################################################
# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
# #
# This software was partially supported by the #
# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
# #
# This software was partially supported by the #
# ADA-FS project under the SPPEXA project funded by the DFG. #
# #
# SPDX-License-Identifier: MIT #
################################################################################
COMMON_CURL_FLAGS="--silent --fail --show-error --location -O"
COMMON_GIT_FLAGS="--quiet --single-branch"
PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PATCH_DIR="${PATCH_DIR}/patches"
DEPENDENCY=""
NA_LAYER=""
DEP_CONFIG=""
VERBOSE=false
Frederic Schimmelpfennig
committed
VALID_DEP_OPTIONS="mogongpu mogon2 mogon1 ngio direct all ci"
MOGONGPU_DEPS=(
"zstd" "lz4" "snappy" "capstone" "mercury" "argobots" "margo" "rocksdb"
"syscall_intercept" "date"
)
MOGON1_DEPS=(
"zstd" "lz4" "snappy" "capstone" "ofi-verbs" "mercury" "argobots" "margo" "rocksdb"
"syscall_intercept" "date"
"bzip2" "zstd" "lz4" "snappy" "capstone" "ofi-experimental" "mercury" "argobots" "margo" "rocksdb-experimental"
"syscall_intercept" "date" "psm2"
NGIO_DEPS=(
"zstd" "lz4" "snappy" "capstone" "ofi-experimental" "mercury" "argobots" "margo" "rocksdb"
"syscall_intercept" "date" "psm2" "agios"
DIRECT_DEPS=(
"ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date"
"bzip2" "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb"
"syscall_intercept" "date" "agios"
CI_DEPS=(
"ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios"
)
# Stop all backround jobs on interruption.
# "kill -- -$$" sends a SIGTERM to the whole process group,
# thus killing also descendants.
# Use single quotes, otherwise this expands now rather than when signalled.
# See shellcheck SC2064.
trap 'trap - SIGTERM && kill -- -$$' SIGINT SIGTERM
exit_child() {
if [ ! $? -eq 0 ]; then
# notify the parent
echo "$1" >&2 ## Send message to stderr. Exclude >&2 if you don't want it that way.
exit "${2:-1}" ## Return a code specified by $2 or 1 by default.
list_dependencies() {
echo "Available dependencies: "
Frederic Schimmelpfennig
committed
echo -n " Mogon GPU: "
for d in "${MOGONGPU_DEPS[@]}"; do
echo -n "$d "
done
echo -n " Mogon 1: "
for d in "${MOGON1_DEPS[@]}"; do
echo -n "$d "
done
echo -n " Mogon 2: "
for d in "${MOGON2_DEPS[@]}"; do
echo -n " NGIO: "
for d in "${NGIO_DEPS[@]}"; do
echo -n "$d "
done
echo -n " Direct GekkoFS dependencies: "
for d in "${DIRECT_DEPS[@]}"; do
echo -n " All: "
for d in "${ALL_DEPS[@]}"; do
echo
echo -n " ci: "
for d in "${CI_DEPS[@]}"; do
echo -n "$d "
done
check_dependency() {
local DEP=$1
shift
local DEP_CONFIG=("$@")
# ignore template when specific dependency is set
if [[ -n "${DEPENDENCY}" ]]; then
# check if specific dependency was set and return from function
if echo "${DEPENDENCY}" | grep -q "${DEP}"; then
return
else
# if not check if dependency is part of dependency config
for e in "${DEP_CONFIG[@]}"; do
if [[ "${DEP}" == "${e}" ]]; then
return
fi
done
fi
false
if [[ "$VERBOSE" == true ]]; then
set -ex
else
set -e
fi
if [[ -d "${SOURCE}/${FOLDER}/.git" ]]; then
cd "${SOURCE}/${FOLDER}" && git fetch -q
git clone ${COMMON_GIT_FLAGS} ${GIT_FLAGS} -- "${REPO}" "${SOURCE}/${FOLDER}"
cd "${SOURCE}/${FOLDER}" && git checkout -qf ${COMMIT}
echo "${ACTION} '${REPO}' to '${FOLDER}' with commit '[${COMMIT}]' and flags '${GIT_FLAGS}'"
if [[ -n "${PATCH}" ]]; then
git apply --verbose "${PATCH_DIR}/${PATCH}"
if [[ "$VERBOSE" == true ]]; then
set -ex
else
set -e
fi
if [[ -d "${SOURCE}/${FOLDER}" ]]; then
# SC2115 Use "${var:?}" to ensure this never expands to /* .
rm -rf "${SOURCE:?}/${FOLDER:?}"
mkdir -p "${SOURCE}/${FOLDER}"
cd "${SOURCE}"
if [[ -f "${SOURCE}/$FILENAME" ]]; then
curl ${COMMON_CURL_FLAGS} "$URL" || error_exit "Failed to download ${URL}" $?
tar -xf "$FILENAME" --directory "${SOURCE}/${FOLDER}" --strip-components=1
echo "Downloaded '${URL}' to '${FOLDER}'"
usage: dl_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CONFIG>] [-d <DEPENDENCY>]
source_path
"
}
help_msg() {
usage_short
This script gets all GekkoFS dependency sources (excluding the fs itself)
positional arguments:
source_path path where the dependency downloads are put
optional arguments:
-h, --help shows this help message and exits
list dependencies available for download with descriptions
network layer that is used for communication. Valid: {bmi,ofi,all}
defaults to 'ofi'
-c <CONFIG>, --config <CONFIG>
allows additional configurations, e.g., for specific clusters
supported values: {mogon2, mogon1, ngio, direct, all, ci}
defaults to 'direct'
-d <DEPENDENCY>, --dependency <DEPENDENCY>
download a specific dependency and ignore --config setting. If unspecified
all dependencies are built and installed based on set --config setting.
Multiple dependencies are supported: Pass a space-separated string (e.g., \"ofi mercury\"
-v, --verbose Increase download verbosity
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
while [[ $# -gt 0 ]]; do
key="$1"
case ${key} in
-n | --na)
NA_LAYER="$2"
shift # past argument
shift # past value
;;
-c | --config)
if [[ -z "$2" ]]; then
echo "ERROR: Missing argument for -c/--config option"
exit 1
fi
if ! echo "$VALID_DEP_OPTIONS" | grep -q "$2"; then
echo "ERROR: Invalid argument for -c/--config option"
exit 1
fi
TMP_DEP_CONF="$2"
shift # past argument
shift # past value
;;
-d | --dependency)
if [[ -z "$2" ]]; then
echo "ERROR: Missing argument for -d/--dependency option"
exit 1
fi
DEPENDENCY="$2"
shift # past argument
shift # past value
;;
-l | --list-dependencies)
list_dependencies
;;
-h | --help)
help_msg
exit
;;
-v | --verbose)
VERBOSE=true
shift # past argument
;;
*) # unknown option
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
set -- "${POSITIONAL[@]}" # restore positional parameters
# positional arguments
if [[ -z ${1+x} ]]; then
echo "ERROR: Positional arguments missing."
SOURCE="$(readlink -mn "${1}")"
if [[ "${NA_LAYER}" == "" ]]; then
echo "Defaulting NAPLUGIN to 'ofi'"
NA_LAYER="ofi"
# enable predefined dependency template
case ${TMP_DEP_CONF} in
Frederic Schimmelpfennig
committed
mogongpu)
DEP_CONFIG=("${MOGONGPU_DEPS[@]}")
[[ -z "${DEPENDENCY}" ]] && echo "'Mogon1' dependencies are downloaded"
;;
DEP_CONFIG=("${MOGON1_DEPS[@]}")
[[ -z "${DEPENDENCY}" ]] && echo "'Mogon1' dependencies are downloaded"
;;
DEP_CONFIG=("${MOGON2_DEPS[@]}")
[[ -z "${DEPENDENCY}" ]] && echo "'Mogon2' dependencies are downloaded"
;;
DEP_CONFIG=("${NGIO_DEPS[@]}")
[[ -z "${DEPENDENCY}" ]] && echo "'NGIO' dependencies are downloaded"
;;
DEP_CONFIG=("${ALL_DEPS[@]}")
[[ -z "${DEPENDENCY}" ]] && echo "'All' dependencies are downloaded"
;;
DEP_CONFIG=("${CI_DEPS[@]}")
[[ -z "${DEPENDENCY}" ]] && echo "'CI' dependencies are downloaded"
;;
direct | *)
DEP_CONFIG=("${DIRECT_DEPS[@]}")
[[ -z "${DEPENDENCY}" ]] && echo "'Direct' GekkoFS dependencies are downloaded (default)"
;;
if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then
echo NAPLUGIN = "${NA_LAYER}"
echo "ERROR: No valid plugin selected"
echo "Source path is set to \"${SOURCE}\""
echo "------------------------------------"
mkdir -p "${SOURCE}"
## Third party dependencies
# get zstd for fast compression in rocksdb
if check_dependency "zstd" "${DEP_CONFIG[@]}"; then
wgetdeps "zstd" "https://github.com/facebook/zstd/archive/v1.3.2.tar.gz" &
fi
# get zlib for rocksdb
if check_dependency "lz4" "${DEP_CONFIG[@]}"; then
wgetdeps "lz4" "https://github.com/lz4/lz4/archive/v1.8.0.tar.gz" &
fi
# get snappy for rocksdb
if check_dependency "snappy" "${DEP_CONFIG[@]}"; then
wgetdeps "snappy" "https://github.com/google/snappy/archive/1.1.7.tar.gz" &
# get bzip2 for rocksdb
if check_dependency "bzip2" "${DEP_CONFIG[@]}"; then
wgetdeps "bzip2" "https://sourceforge.net/projects/bzip2/files/bzip2-1.0.6.tar.gz" &
fi
# get capstone for syscall-intercept
if check_dependency "capstone" "${DEP_CONFIG[@]}"; then
wgetdeps "capstone" "https://github.com/aquynh/capstone/archive/4.0.1.tar.gz" &
fi
## Direct GekkoFS dependencies
if check_dependency "bmi" "${DEP_CONFIG[@]}"; then
if [ "${NA_LAYER}" == "bmi" ] || [ "${NA_LAYER}" == "all" ]; then
clonedeps "bmi" "https://xgitlab.cels.anl.gov/sds/bmi.git" "81ad0575fc57a69269a16208417cbcbefa51f9ea" &
fi
if [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then
if check_dependency "ofi-experimental" "${DEP_CONFIG[@]}"; then
clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.9.1" &
elif check_dependency "ofi-verbs" "${DEP_CONFIG[@]}"; then
# libibverbs 1.2.1-1 used on mogon 1i (installed on system) which is linked to libfabric
# libfabric 1.8 random RPCs fail to be send. 1.9 RPC client cannot be started when in an MPI environment
clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.7.2" &
elif check_dependency "ofi" "${DEP_CONFIG[@]}"; then
clonedeps "libfabric" "https://github.com/ofiwg/libfabric.git" "" "-b v1.8.1" &
if check_dependency "psm2" "${DEP_CONFIG[@]}"; then
wgetdeps "psm2" "https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz" &
fi
if check_dependency "mercury" "${DEP_CONFIG[@]}"; then
clonedeps "mercury" "https://github.com/mercury-hpc/mercury" "41caa143a07ed179a3149cac4af0dc7aa3f946fd" "--recurse-submodules" &
if check_dependency "argobots" "${DEP_CONFIG[@]}"; then
wgetdeps "argobots" "https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz" &
fi
if check_dependency "margo" "${DEP_CONFIG[@]}"; then
clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "v0.6.3" &
if check_dependency "rocksdb-experimental" "${DEP_CONFIG[@]}"; then
wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz" &
elif check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then
wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz" &
if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then
clonedeps "syscall_intercept" "https://github.com/pmem/syscall_intercept.git" "f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" "" "syscall_intercept.patch" &
# get AGIOS
if check_dependency "agios" "${DEP_CONFIG[@]}"; then
clonedeps "agios" "https://github.com/francielizanon/agios.git" "c26a6544200f823ebb8f890dd94e653d148bf226" "-b development" &
if check_dependency "date" "${DEP_CONFIG[@]}"; then
clonedeps "date" "https://github.com/HowardHinnant/date.git" "e7e1482087f58913b80a20b04d5c58d9d6d90155" &
fi
# Wait for all download to be completed