Commit 3e886a85 authored by Alberto Miranda's avatar Alberto Miranda ♨️
Browse files

Merge branch... 

Merge branch 'amiranda/176-using-d-option-in-dl_dep-sh-ends-up-downloading-all-dependencies' into 'master'

Resolve "Various issues with dependency scripts"

This MR adds some tests for the scripts based on [bats-core](https://github.com/bats-core/bats-core), and enables its automatic execution by the CI.

It also changes the separator for dependency flavors from `:` to `%` to prevent clashing with the version separator for profiles (e.g. `libfabric:experimental@mogon2:latest` was ambiguous to `bash`).

Two new bash functions `determine_compiler()` and `profile_has_dependency()` are added to `compile_dep.sh` that are made available to `install` scripts:

- The `determine_compiler()` function makes its best to determine the default compiler available in the system. Installation scripts can access this information via the `COMPILER_NAME`, `COMPILER_FULL_VERSION`, and `COMPILER_MAJOR_VERSION` variables. 
- The `profile_has_dependency()` function allows installation scripts to check whether another dependency is provided by the currently loaded profile, and act accordingly. The function accepts a bash regex with the search argument.

Additionally, this MR fixes the following problems in `dl_dep.sh` and `compile_dep.sh`:

- [x] Using `-d` option in `dl_dep.sh` ends up downloading all dependencies (#176)
- [x] Specifying a full profile in `dl_dep.sh` with `-d` does not work (#177)
- [x] `dl_deps.sh` can download the same dependency in several versions but unpacks them into the same directory (#178)
- [x] ~~rocksdb install of newer versions needs PREFIX for `make install` to specify install path (#180)~~ (Moving to CMake for 0.9.0
- [x] When `clonedeps` is using a tag/branchname to clone, e.g., `HEAD@v1.12.1`, `clonedeps_args` for the dependency needs explicit space, e.g., `["mercury"]=" --recurse-submodules"` otherwise the script fails.
- [x] Dependency patches are being skipped (#184).
- [x] Script should use script directory as `pwd` instead of `bash_pwd`?


Closes #176 #177 #178 #180 #184

See merge request !111
parents d4896773 25fd2f35

.gitignore

+2 −2
Original line number Diff line number Diff line
@@ -25,7 +25,7 @@ 


# Executables

*.exe

*.out

#*.out

*.app



# ---> C
@@ -53,7 +53,7 @@ 


# Executables

*.exe

*.out

#*.out

*.app

*.i*86

*.x86_64

.gitlab-ci.yml

+19 −0
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ variables: 
  INTEGRATION_TESTS_BIN_PATH:   "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration"

  COVERAGE_PATH:                "${CI_PROJECT_DIR}/gkfs/build/.coverage"

  PYTEST:                       "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test"

  BATS:                         "${CI_PROJECT_DIR}/tests/scripts/bats/bin/bats"

  LD_LIBRARY_PATH:              "${CI_PROJECT_DIR}/deps/install/lib:${CI_PROJECT_DIR}/deps/install/lib64"

  # Configuration variables

  GKFS_LOG_LEVEL:               "100"
@@ -97,6 +98,24 @@ gkfwd: 
## Testing

################################################################################



## == tests for scripts ====================

scripts:

  stage: test

  image: gekkofs/testing:0.8.0

  needs: []

  parallel:

    matrix:

      - SUBTEST: [ dl_dep.sh, compile_dep.sh ]

  script:

    - mkdir -p ${BUILD_PATH}/tests/scripts

    - cd ${BUILD_PATH}/tests/scripts

    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/${SUBTEST} --formatter junit > report.xml

  artifacts:

    expire_in: 1 week

    reports:

      junit: ${BUILD_PATH}/tests/scripts/report.xml





## == integration tests for gkfs ===========

gkfs:integration:

  stage: test

.gitmodules

+12 −0
Original line number Diff line number Diff line
@@ -4,3 +4,15 @@ 
[submodule "external/fmt"]

	path = external/fmt

	url = https://github.com/fmtlib/fmt

[submodule "tests/scripts/bats"]

	path = tests/scripts/bats

	url = https://github.com/bats-core/bats-core.git

[submodule "tests/scripts/helpers/bats-assert"]

	path = tests/scripts/helpers/bats-assert

	url = https://github.com/bats-core/bats-assert.git

[submodule "tests/scripts/helpers/bats-support"]

	path = tests/scripts/helpers/bats-support

	url = https://github.com/bats-core/bats-support.git

[submodule "tests/scripts/helpers/bats-file"]

	path = tests/scripts/helpers/bats-file

	url = https://github.com/bats-core/bats-file.git

scripts/compile_dep.sh

+153 −30
Original line number Diff line number Diff line
@@ -27,19 +27,21 @@ 
# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################



PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

PATCH_DIR="${PATCH_DIR}/patches"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

DEPENDENCY=""

CORES=""

SOURCE_DIR=""

INSTALL_DIR=""

PERFORM_TEST=



EXECUTION_MODE=

DRY_RUN=false



DEFAULT_PROFILE="default"

DEFAULT_VERSION="latest"

PROFILE_NAME=${DEFAULT_PROFILE}

PROFILE_VERSION=${DEFAULT_VERSION}

PROFILES_DIR="${PWD}/profiles"

PROFILES_DIR="${SCRIPT_DIR}/profiles"

declare -a PROFILE_DEP_LIST

declare -A PROFILE_DEP_NAMES

declare -A PROFILE_WGETDEPS PROFILE_CLONEDEPS PROFILE_SOURCES PROFILE_EXTRA_INSTALL_ARGS
@@ -47,11 +49,12 @@ declare -A PROFILE_CLONEDEPS_ARGS PROFILE_CLONEDEPS_PATCHES 


usage_short() {

	echo "

usage: compile_dep.sh [-h]

                      [-l [PROFILE_NAME:[VERSION]]]

                      [-p PROFILE_NAME[:VERSION]]

                      [-d DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]]

                      [-j COMPILE_CORES]

usage: compile_dep.sh -h |

                      -p PROFILE_NAME[:PROFILE_VERSION] |

                      -d DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] |

                      -l [PROFILE_NAME:[PROFILE_VERSION]] |

                      -h

                      [ -P PROFILES_DIR ] [ -j COMPILE_CORES] [ -n ]

                      SOURCES_PATH INSTALL_PATH

	"

}
@@ -68,30 +71,43 @@ positional arguments: 




optional arguments:

    -h, --help  shows this help message and exits

    -l, --list-dependencies

                list dependencies available for building and installation

    -p, --profile PROFILE_NAME[:VERSION]

                allows installing a pre-defined set of dependencies as defined

                in ${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to

    -h, --help  Shows this help message and exits

    -l, --list-dependencies [[PROFILE_NAME:]PROFILE_VERSION]

                List dependencies available for building and installation

    -p, --profile PROFILE_NAME[:PROFILE_VERSION]

                Allows installing a pre-defined set of dependencies as defined

                in \${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to

                deploy specific library versions and/or configurations,

                using a recognizable name. Optionally, PROFILE_NAME may include

                a specific version for the profile, e.g. 'mogon2:latest' or

                'ngio:0.8.0', which will download the dependencies defined for

                that specific version. If unspecified, the 'default:latest' profile

                will be used, which should include all the possible dependencies.

    -d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]]

                build and install a specific dependency, ignoring any --profile

    -d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]]

                Build and install a specific dependency, ignoring any --profile

                option provided. If PROFILE_NAME is unspecified, the 'default'

                profile will be used. Similarly, if VERSION is unspecified, the

                'latest' version of the specified profile will be used.

                profile will be used. Similarly, if PROFILE_VERSION is

                unspecified, the 'latest' version of the specified profile will

                be used.

    -j, --compilecores COMPILE_CORES

                number of cores that are used to compile the dependencies

                defaults to number of available cores

                Set the number of cores that will be used to compile the 

                dependencies. If unspecified, defaults to the number of 

                available cores.

    -t, --test  Perform libraries tests.

    -P, --profiles-dir PROFILES_DIR

                Choose the directory to be used when searching for profiles.

                If unspecified, PROFILES_DIR defaults to \${PWD}/profiles.

    -n, --dry-run

                Do not actually run, print only what would be done.

"

}



exec_mode_error() {

    echo "ERROR: --profile and --dependency options are mutually exclusive"

    usage_short

    exit 1

}



list_versions() {



    if [[ ! -d "${PROFILES_DIR}" ]]; then
@@ -267,6 +283,88 @@ find_cmake() { 
    echo "${CMAKE}"

}



determine_compiler() {



    compiler_is_gnu() {

        COMPILER_NAME="g++"



        if ! COMPILER_FULL_VERSION="$(g++ -dumpfullversion 2>&1)"; then

            echo -e "ERROR: Failed to determine compiler version."

            echo -e ">> ${COMPILER_FULL_VERSION}"

            exit 1

        fi



        COMPILER_MAJOR_VERSION="${COMPILER_FULL_VERSION%%.*}"

    }



    compiler_is_clang() {

        COMPILER_NAME="clang"



        if ! COMPILER_FULL_VERSION="$(clang -dumpversion 2>&1)"; then

            echo -e "ERROR: Failed to determine compiler version."

            echo -e ">> ${COMPILER_FULL_VERSION}"

            exit 1

        fi



        COMPILER_MAJOR_VERSION="${COMPILER_FULL_VERSION%%.*}"

    }



    # We honor the CXX environment variable if defined.

    # Otherwise, we try to find the compiler by using `command -v`.

    if [[ -n "${CXX}" && ! "${CXX}" =~ ^(g\+\+|clang)$ ]]; then

        echo "ERROR: Unknown compiler '${CXX}'"

        exit 1

    fi



    if [[ -n "${CXX}" && "${CXX}" =~ ^g\+\+$ ]]; then

        compiler_is_gnu

    elif [[ -n "${CXX}" && "$CXX" =~ ^clang$ ]]; then

        compiler_is_clang

    elif [[ $(command -v g++) ]]; then

        compiler_is_gnu

    elif [[ $(command -v clang) ]]; then

        compiler_is_clang

    else

        echo "ERROR: Unable to determine compiler."

        exit 1

    fi

}



# Check whether the loaded profile defines a particular dependency name.

# The function requires a valid bash regex argument to do the search. The

# function is meant to be used in a conditional context.

#

# Examples:

#   1. Check whether any flavor of 'libfabric' is defined by the profile:

#

#     if profile_has_dependency "^libfabric%.*$"; then

#        echo "libfabric found"

#     fi

#

#   2. Check whether a specific flavor of 'libfabric' is defined by the profile:

#

#     if profile_has_dependency "^libfabric%experimental$"; then

#        echo "libfabric.experimental found"

#     fi

profile_has_dependency() {



    if [[ "$#" -ne 1 ]]; then

        >&2 echo "FATAL: Missing argument in profile_has_dependency()"

        exit 1

    fi



    regex="$1"



    for name in "${PROFILE_DEP_LIST[@]}"; do



        if [[ "${name}" =~ ${regex} ]]; then

            return 0

        fi

    done



    return 1

}





POSITIONAL=()

while [[ $# -gt 0 ]]; do
@@ -274,6 +372,9 @@ while [[ $# -gt 0 ]]; do 


    case ${key} in

    -p | --profile)



        [ -n "${EXECUTION_MODE}" ] && exec_mode_error || EXECUTION_MODE='profile'



        if [[ -z "$2" ]]; then

            echo "ERROR: Missing argument for -p/--profile option"

            exit 1
@@ -292,6 +393,8 @@ while [[ $# -gt 0 ]]; do 
        ;;

    -d | --dependency)



        [ -n "${EXECUTION_MODE}" ] && exec_mode_error || EXECUTION_MODE='dependency'



        if [[ -z "$2" ]]; then

            echo "ERROR: Missing argument for -d/--dependency option"

            exit
@@ -345,6 +448,16 @@ while [[ $# -gt 0 ]]; do 
        PERFORM_TEST=true

        shift

        ;;

    -P | --profiles-dir)



        if [[ ! -d "$2" ]]; then

            echo "ERROR: PROFILES_DIR '$2' does not exist or is not a directory."

            exit 1

        fi



        PROFILES_DIR="$2"

        shift

        ;;

    -l | --list-dependencies)

        if [[ -z "$2" ]]; then

            list_versions
@@ -358,6 +471,10 @@ while [[ $# -gt 0 ]]; do 
        exit

        #shift # past argument

        ;;

    -n | --dry-run)

        DRY_RUN=true

        shift

        ;;

    *) # unknown option

        POSITIONAL+=("$1") # save it in an array for later

        shift              # past argument
@@ -398,9 +515,11 @@ CMAKE="${CMAKE} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}" 


echo "Sources download path = ${SOURCE_DIR}"

echo "Installation path = ${INSTALL_DIR}"

echo "Profile name: ${PROFILE_NAME}"

echo "Profile version: ${PROFILE_VERSION}"

echo "------------------------------------"



mkdir -p "${SOURCE_DIR}"

mkdir -p "${SOURCE_DIR}" || exit 1



######### From now on exits on any error ########

set -e
@@ -411,29 +530,33 @@ export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH}" 


if [[ -n "${DEPENDENCY}" && ! -n "${PROFILE_DEP_NAMES[${DEPENDENCY}]}" ]]; then

    echo "Dependency '${DEPENDENCY}' not found in '${PROFILE_NAME}:${PROFILE_VERSION}'"

    exit

    exit 1

fi



for dep in "${PROFILE_DEP_LIST[@]}"; do

for dep_name in "${PROFILE_DEP_LIST[@]}"; do



    if [[ -n "${DEPENDENCY}" && "${dep}" != "${DEPENDENCY}" ]]; then

    # in dependency mode, skip any dependencies != DEPENDENCY

    if [[ -n "${DEPENDENCY}" && "${dep_name}" != "${DEPENDENCY}" ]]; then

        continue

    fi



    install_script="${PROFILES_DIR}/${PROFILE_VERSION}/install/${dep}.install"

    install_script="${PROFILES_DIR}/${PROFILE_VERSION}/install/${dep_name}.install"



    echo -e "\n\n######## Installing:  ${dep} ###############################\n"

    echo -e "\n\n######## Installing:  ${dep_name} ###############################\n"



    if [[ -f "${install_script}" ]]; then

        source "${install_script}"

        [[ "$DRY_RUN" == true ]] || source "${install_script}"

    else

        echo "WARNING: Install script for '${dep}' not found. Skipping."

        echo "WARNING: Install script for '${dep_name}' not found. Skipping."

        continue

    fi



    if [[ "$DRY_RUN" == false ]]; then

        determine_compiler

        pkg_install



        [ "${PERFORM_TEST}" ] && pkg_check

    fi



done

scripts/dl_dep.sh

+160 −71
Original line number Diff line number Diff line
@@ -29,16 +29,22 @@ 


COMMON_CURL_FLAGS="--silent --fail --show-error --location -O"

COMMON_GIT_FLAGS="--quiet --single-branch -c advice.detachedHead=false"

PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

PATCH_DIR="${PATCH_DIR}/patches"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

PATCH_DIR="${SCRIPT_DIR}/patches"

DEPENDENCY=""



EXECUTION_MODE=

VERBOSE=false

DRY_RUN=false



DEFAULT_PROFILE="default"

DEFAULT_VERSION="latest"

PROFILES_DIR="${PWD}/profiles"

PROFILE_NAME=${DEFAULT_PROFILE}

PROFILE_VERSION=${DEFAULT_VERSION}

PROFILES_DIR="${SCRIPT_DIR}/profiles"

SOURCES_FILE="${PROFILES_DIR}/sources.list"

declare -a PROFILE_DEP_NAMES

declare -a PROFILE_DEP_LIST

declare -A PROFILE_DEP_NAMES

declare -A PROFILE_WGETDEPS PROFILE_CLONEDEPS PROFILE_SOURCES

declare -A PROFILE_CLONEDEPS_ARGS PROFILE_CLONEDEPS_PATCHES
@@ -161,6 +167,7 @@ load_profile() { 


    # make sure we are in a known state

    PROFILE_DEP_NAMES=()

    PROFILE_DEP_LIST=()

    PROFILE_CLONEDEPS=()

    PROFILE_CLONEDEPS_ARGS=()

    PROFILE_CLONEDEPS_PATCHES=()
@@ -193,7 +200,8 @@ load_profile() { 


    # propagate results outside of function

    for i in "${!order[@]}"; do

        PROFILE_DEP_NAMES[$i]="${order[${i}]}"

        PROFILE_DEP_LIST[$i]="${order[${i}]}"

        PROFILE_DEP_NAMES["${order[$i]}"]="$i"

    done



    for k in "${!clonedeps[@]}"; do
@@ -211,6 +219,9 @@ load_profile() { 
    for k in "${!wgetdeps[@]}"; do

        PROFILE_WGETDEPS["${k}"]="${wgetdeps[${k}]}"

    done



    # load source URLs for dependencies

    load_sources

}



load_sources() {
@@ -236,28 +247,29 @@ clonedeps() { 
    fi

    trap exit_child EXIT



    local FOLDER=$1

    local REPO=$2

    local COMMIT=$3

    local GIT_FLAGS=$4

    local PATCH=$5

    local FOLDER="$1"

    local REPO="$2"

    local COMMIT="$3"

    local PATCH="$4"

    shift 4

    local GIT_FLAGS=("$@")



    local ACTION



    if [[ -d "${SOURCE_DIR}/${FOLDER}/.git" ]]; then

        cd "${SOURCE_DIR}/${FOLDER}" && git fetch -q

        [[ "$DRY_RUN" == true ]] || (cd "${SOURCE_DIR}/${FOLDER}" && git fetch -q)

        ACTION="Pulled"

    else

        git clone ${COMMON_GIT_FLAGS} ${GIT_FLAGS} -- "${REPO}" "${SOURCE_DIR}/${FOLDER}"

        [[ "$DRY_RUN" == true ]] || (git clone ${COMMON_GIT_FLAGS} "${GIT_FLAGS[@]}" -- "${REPO}" "${SOURCE_DIR}/${FOLDER}")

        ACTION="Cloned"

    fi

    # fix the version

    cd "${SOURCE_DIR}/${FOLDER}" && git checkout -qf "${COMMIT}"

    echo "${ACTION} '${REPO}' to '${FOLDER}' with commit '[${COMMIT}]' and flags '${GIT_FLAGS}'"

    [[ "$DRY_RUN" == true ]] || (cd "${SOURCE_DIR}/${FOLDER}" && git checkout -qf "${COMMIT}")

    echo "${ACTION} '${REPO}' to '${FOLDER}' with commit '[${COMMIT}]' and flags '${GIT_FLAGS[@]}'"



    # apply patch if provided

    if [[ -n "${PATCH}" ]]; then

        git apply --verbose "${PATCH_DIR}/${PATCH}"

        [[ "$DRY_RUN" == true ]] || (cd "${SOURCE_DIR}/${FOLDER}" && git apply --verbose "${PATCH_DIR}/${PATCH}" )

    fi

}
@@ -271,6 +283,8 @@ wgetdeps() { 


    FOLDER=$1

    URL=$2



    if [[ "$DRY_RUN" == false ]]; then

        if [[ -d "${SOURCE_DIR}/${FOLDER}" ]]; then

            # SC2115 Use "${var:?}" to ensure this never expands to /* .

            rm -rf "${SOURCE_DIR:?}/${FOLDER:?}"
@@ -284,15 +298,17 @@ wgetdeps() { 
        curl ${COMMON_CURL_FLAGS} "$URL" || error_exit "Failed to download ${URL}" $?

        tar -xf "$FILENAME" --directory "${SOURCE_DIR}/${FOLDER}" --strip-components=1

        rm -f "$FILENAME"

    fi

    echo "Downloaded '${URL}' to '${FOLDER}'"

}



usage_short() {

    echo "

usage: dl_dep.sh [-h]

                 [-l [[PROFILE_NAME:]VERSION]]

                 [-p PROFILE_NAME[:VERSION]]

                 [-d DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]]

usage: dl_dep.sh -p PROFILE_NAME[:PROFILE_VERSION] |

                 -d DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] |

                 -l [[PROFILE_NAME:]PROFILE_VERSION] |

                 -h

                 [ -P PROFILES_DIR ] [ -n ]

                 DESTINATION_PATH

	"

}
@@ -304,43 +320,53 @@ help_msg() { 
This script gets all GekkoFS dependency sources (excluding the fs itself)



positional arguments:

        DESTINATION_PATH        path where dependencies should be downloaded

        DESTINATION_PATH        Path where dependencies should be downloaded





optional arguments:

        -h, --help              shows this help message and exits

        -l, --list-dependencies [[PROFILE_NAME:]VERSION]

                                list dependency configuration profiles available for download

        -p, --profile PROFILE_NAME[:VERSION]

                                allows downloading a pre-defined set of dependencies as defined

                                in ${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to 

        -h, --help              Shows this help message and exits

        -l, --list-dependencies [[PROFILE_NAME:]PROFILE_VERSION]

                                List dependency configuration profiles available for download

        -p, --profile PROFILE_NAME[:PROFILE_VERSION]

                                Allows downloading a pre-defined set of dependencies as defined

                                in \${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to 

                                deploy specific library versions and/or configurations,

                                using a recognizable name. Optionally, PROFILE_NAME may include

                                a specific version for the profile, e.g. 'mogon2:latest' or

                                'ngio:0.8.0', which will download the dependencies defined for

                                that specific version. If unspecified, the 'default:latest' profile

                                will be used, which should include all the possible dependencies.

        -d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]]

                                build and install a specific dependency, ignoring any --profile

        -d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]]

                                Build and install a specific dependency, ignoring any --profile

                                option provided. If PROFILE_NAME is unspecified, the 'default'

                                profile will be used. Similarly, if VERSION is unspecified, the

                                'latest' version of the specified profile will be used.

                                profile will be used. Similarly, if PROFILE_VERSION is unspecified,

                                the 'latest' version of the specified profile will be used.

        -P, --profiles-dir PROFILES_DIR

                                Choose the directory to be used when searching for profiles.

                                If unspecified, PROFILES_DIR defaults to \${PWD}/profiles.

        -n, --dry-run           Do not actually run, print only what would be done.

        -v, --verbose           Increase download verbosity

        "

}



exec_mode_error() {

    echo "ERROR: --profile and --dependency options are mutually exclusive"

    usage_short

    exit 1

}



# load default profile for now, might be overridden later

load_profile "${DEFAULT_PROFILE}" "${DEFAULT_VERSION}"



# load source URLs for dependencies

load_sources



POSITIONAL=()

while [[ $# -gt 0 ]]; do

    key="$1"



    case ${key} in

    -p | --profile)



        [ -n "${EXECUTION_MODE}" ] && exec_mode_error || EXECUTION_MODE='profile'



        if [[ -z "$2" ]]; then

            echo "ERROR: Missing argument for -p/--profile option"

            exit 1
@@ -354,17 +380,52 @@ while [[ $# -gt 0 ]]; do 
            PROFILE_VERSION="${DEFAULT_VERSION}"

        fi



        load_profile "${PROFILE_NAME}" "${PROFILE_VERSION}"

        shift # past argument

        shift # past value

        ;;



    -d | --dependency)



        [ -n "${EXECUTION_MODE}" ] && exec_mode_error || EXECUTION_MODE='dependency'



        if [[ -z "$2" ]]; then

            echo "ERROR: Missing argument for -d/--dependency option"

            exit 1

        fi



        # e.g. mercury@mogon1:latest

        if [[ "$2" =~ ^(.*)@(.*):(.*)$ ]]; then

            if [[ -n "${BASH_REMATCH[1]}"  ]]; then

                DEPENDENCY="${BASH_REMATCH[1]}"

            fi



            if [[ -n "${BASH_REMATCH[2]}" ]]; then

                PROFILE_NAME="${BASH_REMATCH[2]}"

            fi



            if [[ -n "${BASH_REMATCH[3]}" ]]; then

                PROFILE_VERSION="${BASH_REMATCH[3]}"

            fi



        # e.g. mercury@mogon1

        elif [[ "$2" =~ ^(.*)@(.*)$ ]]; then

            if [[ -n "${BASH_REMATCH[1]}"  ]]; then

                DEPENDENCY="${BASH_REMATCH[1]}"

            fi



            if [[ -n "${BASH_REMATCH[2]}"  ]]; then

                PROFILE_NAME="${BASH_REMATCH[2]}"

            fi

        # e.g. mercury

        else

            DEPENDENCY="$2"

        fi



        if [[ ! -n "${DEPENDENCY}" ]]; then

            echo "ERROR: Missing dependency name."

            exit 1

        fi



        shift # past argument

        shift # past value

        ;;
@@ -386,6 +447,21 @@ while [[ $# -gt 0 ]]; do 
        VERBOSE=true

        shift # past argument

        ;;

    -P | --profiles-dir)



        if [[ ! -d "$2" ]]; then

            echo "ERROR: PROFILES_DIR '$2' does not exist or is not a directory."

            exit 1

        fi



        PROFILES_DIR="$2"

        SOURCES_FILE="${PROFILES_DIR}/sources.list"

        shift

        ;;

    -n | --dry-run)

        DRY_RUN=true

        shift

        ;;

    *) # unknown option

        POSITIONAL+=("$1") # save it in an array for later

        shift              # past argument
@@ -402,72 +478,85 @@ if [[ -z ${1+x} ]]; then 
fi

SOURCE_DIR="$(readlink -mn "${1}")"



load_profile "${PROFILE_NAME}" "${PROFILE_VERSION}"



if [[ -n "${DEPENDENCY}" && ! -n "${PROFILE_DEP_NAMES[${DEPENDENCY}]}" ]]; then

    echo "ERROR: '${DEPENDENCY}' not found in '${PROFILE_NAME}:${PROFILE_VERSION}'"

    exit 1

fi



echo "Destination path is set to  \"${SOURCE_DIR}\""

echo "Profile name: ${PROFILE_NAME}"

echo "Profile version: ${PROFILE_VERSION}"

echo "------------------------------------"



mkdir -p "${SOURCE_DIR}"

mkdir -p "${SOURCE_DIR}" || exit 1



## download dependencies

for dep in "${PROFILE_DEP_NAMES[@]}"; do

for dep_name in "${PROFILE_DEP_LIST[@]}"; do



    if [[ ! -z "${PROFILE_WGETDEPS[${dep}]:-}" ]]; then

    # in dependency mode, skip any dependencies != DEPENDENCY

    if [[ -n "${DEPENDENCY}" && "${dep_name}" != "${DEPENDENCY}" ]]; then

        continue

    fi



        # dependency names can include a TAG after a colon (e.g. ofi:verbs),

        # remove it

        dep_id=${dep%%:*}

    if [[ ! -z "${PROFILE_WGETDEPS[${dep_name}]:-}" ]]; then



        # find required version for dependency

        dep_version="${PROFILE_WGETDEPS[${dep}]}"

        dep_version="${PROFILE_WGETDEPS[${dep_name}]}"



        # build URL for dependency

        dep_url="${PROFILE_SOURCES[${dep_id}]}"

        dep_url="${PROFILE_SOURCES[${dep_name}]}"



        if [[ -z "${dep_url}" ]]; then

            echo "Missing source URL for '${dep_id}'. Verify ${SOURCES_FILE}."

            echo "Missing source URL for '${dep_name}'. Verify ${SOURCES_FILE}."

            wait

            exit 1

        fi



        dep_url="${dep_url/\{\{VERSION\}\}/${dep_version}}"



        wgetdeps "${dep_id}" "${dep_url}" &



    elif [[ ! -z "${PROFILE_CLONEDEPS[${dep}]:-}" ]]; then

        wgetdeps "${dep_name}" "${dep_url}" &



        # dependency names can include a TAG after a colon (e.g. ofi:verbs),

        # remove it

        dep_id=${dep%%:*}

    elif [[ ! -z "${PROFILE_CLONEDEPS[${dep_name}]:-}" ]]; then



        dep_args=""

        dep_args=()



        # find required version for dependency

        dep_version="${PROFILE_CLONEDEPS[${dep}]}"

        dep_version="${PROFILE_CLONEDEPS[${dep_name}]}"



        # version may be a commit hash, a tag or something like HEAD@BRANCH_NAME

        # if it's the latter, remove the @BRANCH_NAME

        if [[ "${dep_version}" =~ ^(.*)@(.*)$ ]]; then

            dep_args+="-b ${BASH_REMATCH[2]}"

            dep_args+=("--branch=${BASH_REMATCH[2]}")

            dep_version=${BASH_REMATCH[1]}

        fi



        # build URL for dependency

        dep_url="${PROFILE_SOURCES[${dep_id}]}"

        dep_url="${PROFILE_SOURCES[${dep_name}]}"



        if [[ -z "${dep_url}" ]]; then

            echo "Missing source URL for '${dep_id}'. Verify ${SOURCES_FILE}."

            echo "Missing source URL for '${dep_name}'. Verify ${SOURCES_FILE}."

            wait

            exit 1

        fi



        dep_url="${dep_url/\{\{VERSION\}\}/${dep_version}}"



        # check if extra args are required

        dep_args+="${PROFILE_CLONEDEPS_ARGS[${dep}]}"

        for arg in "${PROFILE_CLONEDEPS_ARGS[${dep_name}]}";

        do

            if [[ -n "${arg}" ]]; then

                dep_args+=("${arg}")

            fi

        done



        dep_patch=${PROFILE_CLONEDEPS_PATCHES[${dep}]}

        dep_patch=${PROFILE_CLONEDEPS_PATCHES[${dep_name}]}



        clonedeps "${dep}" "${dep_url}" "${dep_version}" "${dep_args}" "${dep_patch}" &

        clonedeps "${dep_name}" "${dep_url}" "${dep_version}" "${dep_patch}" "${dep_args[@]}" &



    else

        echo "Unknown dependency '${dep}'."

        echo "Unknown dependency '${dep_name}'."

        exit 1

    fi

done