Newer
Older
################################################################################
# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
# #
# This software was partially supported by the #
# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
# #
# This software was partially supported by the #
# ADA-FS project under the SPPEXA project funded by the DFG. #
# #
# This file is part of GekkoFS. #
# #
# GekkoFS is free software: you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation, either version 3 of the License, or #
# (at your option) any later version. #
# #
# GekkoFS is distributed in the hope that it will be useful, #
# but WITHOUT ANY WARRANTY; without even the implied warranty of #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
# GNU General Public License for more details. #
# #
# You should have received a copy of the GNU General Public License #
# along with GekkoFS. If not, see <https://www.gnu.org/licenses/>. #
# #
# SPDX-License-Identifier: GPL-3.0-or-later #
################################################################################
COMMON_CURL_FLAGS="--silent --fail --show-error --location -O"
COMMON_GIT_FLAGS="--quiet --single-branch -c advice.detachedHead=false"
PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PATCH_DIR="${PATCH_DIR}/patches"
DEPENDENCY=""
EXECUTION_MODE=
VERBOSE=false
DEFAULT_PROFILE="default"
DEFAULT_VERSION="latest"
PROFILES_DIR="${PWD}/profiles"
SOURCES_FILE="${PROFILES_DIR}/sources.list"
declare -a PROFILE_DEP_LIST
declare -A PROFILE_DEP_NAMES
declare -A PROFILE_WGETDEPS PROFILE_CLONEDEPS PROFILE_SOURCES
declare -A PROFILE_CLONEDEPS_ARGS PROFILE_CLONEDEPS_PATCHES
# Stop all backround jobs on interruption.
# "kill -- -$$" sends a SIGTERM to the whole process group,
# thus killing also descendants.
# Use single quotes, otherwise this expands now rather than when signalled.
# See shellcheck SC2064.
trap 'trap - SIGTERM && kill -- -$$' SIGINT SIGTERM
exit_child() {
if [ ! $? -eq 0 ]; then
# notify the parent
echo "$1" >&2 ## Send message to stderr. Exclude >&2 if you don't want it that way.
exit "${2:-1}" ## Return a code specified by $2 or 1 by default.
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
list_versions() {
if [[ ! -d "${PROFILES_DIR}" ]]; then
echo "Directory '${PROFILES_DIR}' does not exist. No profiles available."
exit 1
fi
declare -A versions
while IFS= read -r -d '' filename; do
id="$(basename $(dirname ${filename}))"
profile="$(basename ${filename%%.specs})"
versions[$id]+="${profile} "
done < <(find -L "${PROFILES_DIR}" -type f -name "*.specs" -print0 | sort -z)
echo -e "Available versions and configuration profiles:\n"
for id in "${!versions[@]}"; do
echo " ${id}:"
echo -e " ${versions[${id}]}\n"
done
exit 0
}
list_profiles() {
local TAG=$1
if [[ "$TAG" =~ ^(.*):(.*)$ ]]; then
PROFILE="${BASH_REMATCH[1]}.specs"
if [[ -n ${BASH_REMATCH[2]} ]]; then
VERSION="${BASH_REMATCH[2]}"
else
VERSION="latest"
fi
else
VERSION="${TAG}"
fi
if [[ ! -d "${PROFILES_DIR}" ]]; then
echo "Directory '${PROFILES_DIR}' does not exist. No configuration profiles found."
exit 1
fi
if [[ ! -d "${PROFILES_DIR}/${VERSION}" ]]; then
echo "Version ${VERSION} does not exist. No configuration profiles found."
exit 1
fi
echo -e "Configuration profiles for '${VERSION}':\n"
find "${PROFILES_DIR}/${VERSION}/${PROFILE}" -type f -name "*.specs" -print0 | sort -z | while IFS= read -r -d '' filename; do
basename=$(basename "${filename}")
version=$(basename $(dirname "${filename}"))
profile="${basename%.*}"
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
echo "* ${profile}:${version} (${filename})"
source "${filename}"
if [[ -n "${comment}" ]]; then
echo -e "\n ${comment}\n"
fi
for d in "${order[@]}";
do
if [[ -n ${wgetdeps[${d}]} ]]; then
echo " ${d}: ${wgetdeps[${d}]}"
elif [[ -n ${clonedeps[${d}]} ]]; then
echo " ${d}: ${clonedeps[${d}]}"
else
echo " ${d}: ???"
fi
done
echo ""
unset wgetdeps
unset clonedeps
unset clonedeps_args
unset clonedeps_patches
unset comment
unset order
exit 0
}
load_profile() {
local profile=$1
local version=$2
shift
# make sure we are in a known state
PROFILE_DEP_NAMES=()
PROFILE_DEP_LIST=()
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
PROFILE_CLONEDEPS=()
PROFILE_CLONEDEPS_ARGS=()
PROFILE_CLONEDEPS_PATCHES=()
PROFILE_WGETDEPS=()
local filename="${PROFILES_DIR}/${version}/${profile}.specs"
if [[ ! -f "${filename}" ]]; then
echo "Profile '${profile}:${version}' does not exist."
exit 1
fi
source "${filename}"
# some checks
if [[ -z "${wgetdeps[*]}" && -z "${clonedeps[*]}" ]]; then
echo "Profile '${profile}' is invalid."
exit 1
fi
if [[ -z "${order[*]}" ]]; then
echo "Profile '${profile}' is invalid."
exit 1
fi
if [[ "$((${#wgetdeps[@]}+${#clonedeps[@]}))" -ne "${#order[@]}" ]]; then
echo "Profile '${profile}' is invalid."
exit 1
fi
# propagate results outside of function
for i in "${!order[@]}"; do
PROFILE_DEP_LIST[$i]="${order[${i}]}"
PROFILE_DEP_NAMES["${order[$i]}"]="$i"
for k in "${!clonedeps[@]}"; do
PROFILE_CLONEDEPS["${k}"]="${clonedeps[${k}]}"
for k in "${!clonedeps_args[@]}"; do
PROFILE_CLONEDEPS_ARGS["${k}"]="${clonedeps_args[${k}]}"
for k in "${!clonedeps_patches[@]}"; do
PROFILE_CLONEDEPS_PATCHES["${k}"]="${clonedeps_patches[${k}]}"
for k in "${!wgetdeps[@]}"; do
PROFILE_WGETDEPS["${k}"]="${wgetdeps[${k}]}"
load_sources() {
if [[ ! -f "${SOURCES_FILE}" ]]; then
echo "Missing dependency sources at '${SOURCES_FILE}'."
exit 1
source "${SOURCES_FILE}"
# propagate sources outside of function
for k in "${!sources[@]}"; do
PROFILE_SOURCES["${k}"]="${sources[${k}]}"
done
if [[ "$VERBOSE" == true ]]; then
set -ex
else
set -e
fi
if [[ -d "${SOURCE_DIR}/${FOLDER}/.git" ]]; then
[[ "$DRY_RUN" == true ]] || (cd "${SOURCE_DIR}/${FOLDER}" && git fetch -q)
[[ "$DRY_RUN" == true ]] || (git clone ${COMMON_GIT_FLAGS} ${GIT_FLAGS} -- "${REPO}" "${SOURCE_DIR}/${FOLDER}")
[[ "$DRY_RUN" == true ]] || (cd "${SOURCE_DIR}/${FOLDER}" && git checkout -qf "${COMMIT}")
echo "${ACTION} '${REPO}' to '${FOLDER}' with commit '[${COMMIT}]' and flags '${GIT_FLAGS}'"
if [[ -n "${PATCH}" ]]; then
[[ "$DRY_RUN" == true ]] || git apply --verbose "${PATCH_DIR}/${PATCH}"
if [[ "$VERBOSE" == true ]]; then
set -ex
else
set -e
fi
if [[ "$DRY_RUN" == false ]]; then
if [[ -d "${SOURCE_DIR}/${FOLDER}" ]]; then
# SC2115 Use "${var:?}" to ensure this never expands to /* .
rm -rf "${SOURCE_DIR:?}/${FOLDER:?}"
fi
mkdir -p "${SOURCE_DIR}/${FOLDER}"
cd "${SOURCE_DIR}"
FILENAME="$(basename $URL)"
if [[ -f "${SOURCE_DIR}/$FILENAME" ]]; then
rm -f "${SOURCE_DIR}/$FILENAME"
fi
curl ${COMMON_CURL_FLAGS} "$URL" || error_exit "Failed to download ${URL}" $?
tar -xf "$FILENAME" --directory "${SOURCE_DIR}/${FOLDER}" --strip-components=1
rm -f "$FILENAME"
echo "Downloaded '${URL}' to '${FOLDER}'"
usage: dl_dep.sh -p PROFILE_NAME[:PROFILE_VERSION] |
-d DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] |
-l [[PROFILE_NAME:]PROFILE_VERSION] | -n | -h
usage_short
This script gets all GekkoFS dependency sources (excluding the fs itself)
DESTINATION_PATH path where dependencies should be downloaded
optional arguments:
-h, --help shows this help message and exits
-l, --list-dependencies [[PROFILE_NAME:]PROFILE_VERSION]
list dependency configuration profiles available for download
-p, --profile PROFILE_NAME[:PROFILE_VERSION]
allows downloading a pre-defined set of dependencies as defined
in \${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to
deploy specific library versions and/or configurations,
using a recognizable name. Optionally, PROFILE_NAME may include
a specific version for the profile, e.g. 'mogon2:latest' or
'ngio:0.8.0', which will download the dependencies defined for
that specific version. If unspecified, the 'default:latest' profile
will be used, which should include all the possible dependencies.
-d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]]
build and install a specific dependency, ignoring any --profile
option provided. If PROFILE_NAME is unspecified, the 'default'
profile will be used. Similarly, if PROFILE_VERSION is unspecified,
the 'latest' version of the specified profile will be used.
-n, --dry-run Do not actually run, print only what would be done.
-v, --verbose Increase download verbosity
exec_mode_error() {
echo "ERROR: --profile and --dependency options are mutually exclusive"
usage_short
exit 1
}
# load default profile for now, might be overridden later
load_profile "${DEFAULT_PROFILE}" "${DEFAULT_VERSION}"
# load source URLs for dependencies
load_sources
while [[ $# -gt 0 ]]; do
key="$1"
case ${key} in
[ -n "${EXECUTION_MODE}" ] && exec_mode_error || EXECUTION_MODE='profile'
if [[ -z "$2" ]]; then
echo "ERROR: Missing argument for -p/--profile option"
exit 1
fi
if [[ "$2" =~ ^(.*):(.*)$ ]]; then
PROFILE_NAME="${BASH_REMATCH[1]}"
PROFILE_VERSION="${BASH_REMATCH[2]}"
else
PROFILE_NAME="$2"
PROFILE_VERSION="${DEFAULT_VERSION}"
shift # past argument
shift # past value
;;
-d | --dependency)
[ -n "${EXECUTION_MODE}" ] && exec_mode_error || EXECUTION_MODE='dependency'
if [[ -z "$2" ]]; then
echo "ERROR: Missing argument for -d/--dependency option"
exit 1
fi
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
PROFILE_NAME=${DEFAULT_PROFILE}
PROFILE_VERSION=${DEFAULT_VERSION}
# e.g. mercury@mogon1:latest
if [[ "$2" =~ ^(.*)@(.*):(.*)$ ]]; then
if [[ -n "${BASH_REMATCH[1]}" ]]; then
DEPENDENCY="${BASH_REMATCH[1]}"
fi
if [[ -n "${BASH_REMATCH[2]}" ]]; then
PROFILE_NAME="${BASH_REMATCH[2]}"
fi
if [[ -n "${BASH_REMATCH[3]}" ]]; then
PROFILE_VERSION="${BASH_REMATCH[3]}"
fi
# e.g. mercury@mogon1
elif [[ "$2" =~ ^(.*)@(.*)$ ]]; then
if [[ -n "${BASH_REMATCH[1]}" ]]; then
DEPENDENCY="${BASH_REMATCH[1]}"
fi
if [[ -n "${BASH_REMATCH[2]}" ]]; then
PROFILE_NAME="${BASH_REMATCH[2]}"
fi
# e.g. mercury
else
DEPENDENCY="$2"
fi
if [[ ! -n "${DEPENDENCY}" ]]; then
echo "ERROR: Missing dependency name."
exit 1
fi
shift # past argument
shift # past value
;;
-l | --list-dependencies)
if [[ -z "$2" ]]; then
list_versions
else
list_profiles "$2"
fi
;;
-h | --help)
help_msg
exit
;;
-v | --verbose)
VERBOSE=true
shift # past argument
;;
-n | --dry-run)
DRY_RUN=true
shift
;;
*) # unknown option
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
set -- "${POSITIONAL[@]}" # restore positional parameters
# positional arguments
if [[ -z ${1+x} ]]; then
echo "ERROR: Positional arguments missing."
load_profile "${PROFILE_NAME}" "${PROFILE_VERSION}"
if [[ -n "${DEPENDENCY}" && ! -n "${PROFILE_DEP_NAMES[${DEPENDENCY}]}" ]]; then
echo "ERROR: '${DEPENDENCY}' not found in '${PROFILE_NAME}:${PROFILE_VERSION}'"
exit 1
fi
echo "Destination path is set to \"${SOURCE_DIR}\""
echo "------------------------------------"
for dep_name in "${PROFILE_DEP_LIST[@]}"; do
# in dependency mode, skip any dependencies != DEPENDENCY
if [[ -n "${DEPENDENCY}" && "${dep_name}" != "${DEPENDENCY}" ]]; then
Alberto Miranda
committed
continue
fi
if [[ ! -z "${PROFILE_WGETDEPS[${dep_name}]:-}" ]]; then
# dependency names can include a TAG after a colon (e.g. ofi:verbs),
# remove it
# find required version for dependency
dep_version="${PROFILE_WGETDEPS[${dep_name}]}"
# build URL for dependency
dep_url="${PROFILE_SOURCES[${dep_id}]}"
if [[ -z "${dep_url}" ]]; then
echo "Missing source URL for '${dep_id}'. Verify ${SOURCES_FILE}."
exit 1
fi
dep_url="${dep_url/\{\{VERSION\}\}/${dep_version}}"
wgetdeps "${dep_id}" "${dep_url}" &
elif [[ ! -z "${PROFILE_CLONEDEPS[${dep_name}]:-}" ]]; then
# dependency names can include a TAG after a colon (e.g. ofi:verbs),
# remove it
# find required version for dependency
dep_version="${PROFILE_CLONEDEPS[${dep_name}]}"
# version may be a commit hash, a tag or something like HEAD@BRANCH_NAME
# if it's the latter, remove the @BRANCH_NAME
if [[ "${dep_version}" =~ ^(.*)@(.*)$ ]]; then
dep_args+="-b ${BASH_REMATCH[2]}"
dep_version=${BASH_REMATCH[1]}
fi
# build URL for dependency
dep_url="${PROFILE_SOURCES[${dep_id}]}"
if [[ -z "${dep_url}" ]]; then
echo "Missing source URL for '${dep_id}'. Verify ${SOURCES_FILE}."
exit 1
fi
dep_url="${dep_url/\{\{VERSION\}\}/${dep_version}}"
# check if extra args are required
dep_args+="${PROFILE_CLONEDEPS_ARGS[${dep_name}]}"
dep_patch=${PROFILE_CLONEDEPS_PATCHES[${dep_name}]}
clonedeps "${dep_name}" "${dep_url}" "${dep_version}" "${dep_args}" "${dep_patch}" &
echo "Unknown dependency '${dep_name}'."
# Wait for all download to be completed