Newer
Older
################################################################################
# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
# #
# This software was partially supported by the #
# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
# #
# This software was partially supported by the #
# ADA-FS project under the SPPEXA project funded by the DFG. #
# #
# This file is part of GekkoFS. #
# #
# GekkoFS is free software: you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation, either version 3 of the License, or #
# (at your option) any later version. #
# #
# GekkoFS is distributed in the hope that it will be useful, #
# but WITHOUT ANY WARRANTY; without even the implied warranty of #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
# GNU General Public License for more details. #
# #
# You should have received a copy of the GNU General Public License #
# along with GekkoFS. If not, see <https://www.gnu.org/licenses/>. #
# #
# SPDX-License-Identifier: GPL-3.0-or-later #
################################################################################
COMMON_CURL_FLAGS="--silent --fail --show-error --location -O"
COMMON_GIT_FLAGS="--quiet --single-branch -c advice.detachedHead=false"
PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PATCH_DIR="${PATCH_DIR}/patches"
DEPENDENCY=""
VERBOSE=false
DEFAULT_PROFILE="default"
DEFAULT_VERSION="latest"
PROFILES_DIR="${PWD}/profiles"
SOURCES_FILE="${PROFILES_DIR}/sources.list"
declare -a PROFILE_DEP_NAMES
declare -A PROFILE_WGETDEPS PROFILE_CLONEDEPS PROFILE_SOURCES
declare -A PROFILE_CLONEDEPS_ARGS PROFILE_CLONEDEPS_PATCHES
# Stop all backround jobs on interruption.
# "kill -- -$$" sends a SIGTERM to the whole process group,
# thus killing also descendants.
# Use single quotes, otherwise this expands now rather than when signalled.
# See shellcheck SC2064.
trap 'trap - SIGTERM && kill -- -$$' SIGINT SIGTERM
exit_child() {
if [ ! $? -eq 0 ]; then
# notify the parent
echo "$1" >&2 ## Send message to stderr. Exclude >&2 if you don't want it that way.
exit "${2:-1}" ## Return a code specified by $2 or 1 by default.
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
list_versions() {
if [[ ! -d "${PROFILES_DIR}" ]]; then
echo "Directory '${PROFILES_DIR}' does not exist. No profiles available."
exit 1
fi
declare -A versions
while IFS= read -r -d '' filename; do
id="$(basename $(dirname ${filename}))"
profile="$(basename ${filename%%.specs})"
versions[$id]+="${profile} "
done < <(find -L "${PROFILES_DIR}" -type f -name "*.specs" -print0 | sort -z)
echo -e "Available versions and configuration profiles:\n"
for id in "${!versions[@]}"; do
echo " ${id}:"
echo -e " ${versions[${id}]}\n"
done
exit 0
}
list_profiles() {
local TAG=$1
if [[ "$TAG" =~ ^(.*):(.*)$ ]]; then
PROFILE="${BASH_REMATCH[1]}.specs"
if [[ -n ${BASH_REMATCH[2]} ]]; then
VERSION="${BASH_REMATCH[2]}"
else
VERSION="latest"
fi
else
VERSION="${TAG}"
fi
if [[ ! -d "${PROFILES_DIR}" ]]; then
echo "Directory '${PROFILES_DIR}' does not exist. No configuration profiles found."
exit 1
fi
if [[ ! -d "${PROFILES_DIR}/${VERSION}" ]]; then
echo "Version ${VERSION} does not exist. No configuration profiles found."
exit 1
fi
echo -e "Configuration profiles for '${VERSION}':\n"
find "${PROFILES_DIR}/${VERSION}/${PROFILE}" -type f -name "*.specs" -print0 | sort -z | while IFS= read -r -d '' filename; do
basename=$(basename "${filename}")
version=$(basename $(dirname "${filename}"))
profile="${basename%.*}"
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
echo "* ${profile}:${version} (${filename})"
source "${filename}"
if [[ -n "${comment}" ]]; then
echo -e "\n ${comment}\n"
fi
for d in "${order[@]}";
do
if [[ -n ${wgetdeps[${d}]} ]]; then
echo " ${d}: ${wgetdeps[${d}]}"
elif [[ -n ${clonedeps[${d}]} ]]; then
echo " ${d}: ${clonedeps[${d}]}"
else
echo " ${d}: ???"
fi
done
echo ""
unset wgetdeps
unset clonedeps
unset clonedeps_args
unset clonedeps_patches
unset comment
unset order
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
exit 0
}
load_profile() {
local profile=$1
local version=$2
shift
# make sure we are in a known state
PROFILE_DEP_NAMES=()
PROFILE_CLONEDEPS=()
PROFILE_CLONEDEPS_ARGS=()
PROFILE_CLONEDEPS_PATCHES=()
PROFILE_WGETDEPS=()
local filename="${PROFILES_DIR}/${version}/${profile}.specs"
if [[ ! -f "${filename}" ]]; then
echo "Profile '${profile}:${version}' does not exist."
exit 1
fi
source "${filename}"
# some checks
if [[ -z "${wgetdeps[*]}" && -z "${clonedeps[*]}" ]]; then
echo "Profile '${profile}' is invalid."
exit 1
fi
if [[ -z "${order[*]}" ]]; then
echo "Profile '${profile}' is invalid."
exit 1
fi
if [[ "$((${#wgetdeps[@]}+${#clonedeps[@]}))" -ne "${#order[@]}" ]]; then
echo "Profile '${profile}' is invalid."
exit 1
fi
# propagate results outside of function
for i in "${!order[@]}"; do
PROFILE_DEP_NAMES[$i]="${order[${i}]}"
for k in "${!clonedeps[@]}"; do
PROFILE_CLONEDEPS["${k}"]="${clonedeps[${k}]}"
for k in "${!clonedeps_args[@]}"; do
PROFILE_CLONEDEPS_ARGS["${k}"]="${clonedeps_args[${k}]}"
for k in "${!clonedeps_patches[@]}"; do
PROFILE_CLONEDEPS_PATCHES["${k}"]="${clonedeps_patches[${k}]}"
for k in "${!wgetdeps[@]}"; do
PROFILE_WGETDEPS["${k}"]="${wgetdeps[${k}]}"
load_sources() {
if [[ ! -f "${SOURCES_FILE}" ]]; then
echo "Missing dependency sources at '${SOURCES_FILE}'."
exit 1
source "${SOURCES_FILE}"
# propagate sources outside of function
for k in "${!sources[@]}"; do
PROFILE_SOURCES["${k}"]="${sources[${k}]}"
done
if [[ "$VERBOSE" == true ]]; then
set -ex
else
set -e
fi
if [[ -d "${SOURCE_DIR}/${FOLDER}/.git" ]]; then
cd "${SOURCE_DIR}/${FOLDER}" && git fetch -q
git clone ${COMMON_GIT_FLAGS} ${GIT_FLAGS} -- "${REPO}" "${SOURCE_DIR}/${FOLDER}"
cd "${SOURCE_DIR}/${FOLDER}" && git checkout -qf "${COMMIT}"
echo "${ACTION} '${REPO}' to '${FOLDER}' with commit '[${COMMIT}]' and flags '${GIT_FLAGS}'"
if [[ -n "${PATCH}" ]]; then
git apply --verbose "${PATCH_DIR}/${PATCH}"
if [[ "$VERBOSE" == true ]]; then
set -ex
else
set -e
fi
if [[ -d "${SOURCE_DIR}/${FOLDER}" ]]; then
# SC2115 Use "${var:?}" to ensure this never expands to /* .
mkdir -p "${SOURCE_DIR}/${FOLDER}"
cd "${SOURCE_DIR}"
if [[ -f "${SOURCE_DIR}/$FILENAME" ]]; then
rm -f "${SOURCE_DIR}/$FILENAME"
curl ${COMMON_CURL_FLAGS} "$URL" || error_exit "Failed to download ${URL}" $?
tar -xf "$FILENAME" --directory "${SOURCE_DIR}/${FOLDER}" --strip-components=1
echo "Downloaded '${URL}' to '${FOLDER}'"
usage: dl_dep.sh [-h]
[-l [[PROFILE_NAME:]VERSION]]
[-p PROFILE_NAME[:VERSION]]
[-d DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]]
DESTINATION_PATH
usage_short
This script gets all GekkoFS dependency sources (excluding the fs itself)
DESTINATION_PATH path where dependencies should be downloaded
optional arguments:
-h, --help shows this help message and exits
-l, --list-dependencies [[PROFILE_NAME:]VERSION]
list dependency configuration profiles available for download
-p, --profile PROFILE_NAME[:VERSION]
allows downloading a pre-defined set of dependencies as defined
in ${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to
deploy specific library versions and/or configurations,
using a recognizable name. Optionally, PROFILE_NAME may include
a specific version for the profile, e.g. 'mogon2:latest' or
'ngio:0.8.0', which will download the dependencies defined for
that specific version. If unspecified, the 'default:latest' profile
will be used, which should include all the possible dependencies.
-d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]]
build and install a specific dependency, ignoring any --profile
option provided. If PROFILE_NAME is unspecified, the 'default'
profile will be used. Similarly, if VERSION is unspecified, the
'latest' version of the specified profile will be used.
-v, --verbose Increase download verbosity
# load default profile for now, might be overridden later
load_profile "${DEFAULT_PROFILE}" "${DEFAULT_VERSION}"
# load source URLs for dependencies
load_sources
while [[ $# -gt 0 ]]; do
key="$1"
case ${key} in
if [[ -z "$2" ]]; then
echo "ERROR: Missing argument for -p/--profile option"
exit 1
fi
if [[ "$2" =~ ^(.*):(.*)$ ]]; then
PROFILE_NAME="${BASH_REMATCH[1]}"
PROFILE_VERSION="${BASH_REMATCH[2]}"
else
PROFILE_NAME="$2"
PROFILE_VERSION="${DEFAULT_VERSION}"
load_profile "${PROFILE_NAME}" "${PROFILE_VERSION}"
shift # past argument
shift # past value
;;
-d | --dependency)
if [[ -z "$2" ]]; then
echo "ERROR: Missing argument for -d/--dependency option"
exit 1
fi
DEPENDENCY="$2"
shift # past argument
shift # past value
;;
-l | --list-dependencies)
if [[ -z "$2" ]]; then
list_versions
else
list_profiles "$2"
fi
;;
-h | --help)
help_msg
exit
;;
-v | --verbose)
VERBOSE=true
shift # past argument
;;
*) # unknown option
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
set -- "${POSITIONAL[@]}" # restore positional parameters
# positional arguments
if [[ -z ${1+x} ]]; then
echo "ERROR: Positional arguments missing."
echo "Destination path is set to \"${SOURCE_DIR}\""
echo "------------------------------------"
## download dependencies
for dep in "${PROFILE_DEP_NAMES[@]}"; do
Alberto Miranda
committed
if [[ -n "${DEPENDENCY}" && "${dep}" != "${DEPENDENCY}" ]]; then
continue
fi
if [[ ! -z "${PROFILE_WGETDEPS[${dep}]:-}" ]]; then
# dependency names can include a TAG after a colon (e.g. ofi:verbs),
# remove it
dep_id=${dep%%:*}
# find required version for dependency
dep_version="${PROFILE_WGETDEPS[${dep}]}"
# build URL for dependency
dep_url="${PROFILE_SOURCES[${dep_id}]}"
if [[ -z "${dep_url}" ]]; then
echo "Missing source URL for '${dep_id}'. Verify ${SOURCES_FILE}."
exit 1
fi
dep_url="${dep_url/\{\{VERSION\}\}/${dep_version}}"
wgetdeps "${dep_id}" "${dep_url}" &
elif [[ ! -z "${PROFILE_CLONEDEPS[${dep}]:-}" ]]; then
# dependency names can include a TAG after a colon (e.g. ofi:verbs),
# remove it
dep_id=${dep%%:*}
# find required version for dependency
dep_version="${PROFILE_CLONEDEPS[${dep}]}"
# version may be a commit hash, a tag or something like HEAD@BRANCH_NAME
# if it's the latter, remove the @BRANCH_NAME
if [[ "${dep_version}" =~ ^(.*)@(.*)$ ]]; then
dep_args+="-b ${BASH_REMATCH[2]}"
dep_version=${BASH_REMATCH[1]}
fi
# build URL for dependency
dep_url="${PROFILE_SOURCES[${dep_id}]}"
if [[ -z "${dep_url}" ]]; then
echo "Missing source URL for '${dep_id}'. Verify ${SOURCES_FILE}."
exit 1
fi
dep_url="${dep_url/\{\{VERSION\}\}/${dep_version}}"
# check if extra args are required
dep_args+="${PROFILE_CLONEDEPS_ARGS[${dep}]}"
dep_patch=${PROFILE_CLONEDEPS_PATCHES[${dep}]}
clonedeps "${dep}" "${dep_url}" "${dep_version}" "${dep_args}" "${dep_patch}" &
else
echo "Unknown dependency '${dep}'."
exit 1
fi
done
# Wait for all download to be completed