From a73831644976fdcd4c09395ad0293f25ad29d64d Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 14 Oct 2021 13:13:36 +0200 Subject: [PATCH 01/19] dep_scripts: Skip dependencies not explicitly provided via -d in dl_dep.sh --- scripts/dl_dep.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 14274ef8d..cbc6a7fb0 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -410,6 +410,10 @@ mkdir -p "${SOURCE_DIR}" ## download dependencies for dep in "${PROFILE_DEP_NAMES[@]}"; do + if [[ -n "${DEPENDENCY}" && "${dep}" != "${DEPENDENCY}" ]]; then + continue + fi + if [[ ! -z "${PROFILE_WGETDEPS[${dep}]:-}" ]]; then # dependency names can include a TAG after a colon (e.g. ofi:verbs), -- GitLab From 58e50243da9ba51200774aa1c00672625e74eb79 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 4 Nov 2021 12:43:19 +0100 Subject: [PATCH 02/19] dep_scripts: Separate dependency and profile execution modes --- scripts/compile_dep.sh | 33 +++++++++++++++++++++++---------- scripts/dl_dep.sh | 33 +++++++++++++++++++++++---------- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 798ea6ecf..e52369f20 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -35,6 +35,8 @@ SOURCE_DIR="" INSTALL_DIR="" PERFORM_TEST= +EXECUTION_MODE= + DEFAULT_PROFILE="default" DEFAULT_VERSION="latest" PROFILE_NAME=${DEFAULT_PROFILE} @@ -47,11 +49,10 @@ declare -A PROFILE_CLONEDEPS_ARGS PROFILE_CLONEDEPS_PATCHES usage_short() { echo " -usage: compile_dep.sh [-h] - [-l [PROFILE_NAME:[VERSION]]] - [-p PROFILE_NAME[:VERSION]] - [-d DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]] - [-j COMPILE_CORES] +usage: compile_dep.sh [ -p PROFILE_NAME[:PROFILE_VERSION] | + -d DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] ] + [ -l [PROFILE_NAME:[PROFILE_VERSION]] ] + [ -j COMPILE_CORES] [ -h ] SOURCES_PATH INSTALL_PATH " } @@ -71,20 +72,21 @@ optional arguments: -h, --help shows this help message and exits -l, --list-dependencies list dependencies available for building and installation - -p, --profile PROFILE_NAME[:VERSION] + -p, --profile PROFILE_NAME[:PROFILE_VERSION] allows installing a pre-defined set of dependencies as defined - in ${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to + in \${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to deploy specific library versions and/or configurations, using a recognizable name. Optionally, PROFILE_NAME may include a specific version for the profile, e.g. 'mogon2:latest' or 'ngio:0.8.0', which will download the dependencies defined for that specific version. If unspecified, the 'default:latest' profile will be used, which should include all the possible dependencies. - -d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]] + -d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] build and install a specific dependency, ignoring any --profile option provided. If PROFILE_NAME is unspecified, the 'default' - profile will be used. Similarly, if VERSION is unspecified, the - 'latest' version of the specified profile will be used. + profile will be used. Similarly, if PROFILE_VERSION is + unspecified, the 'latest' version of the specified profile will + be used. -j, --compilecores COMPILE_CORES number of cores that are used to compile the dependencies defaults to number of available cores @@ -92,6 +94,12 @@ optional arguments: " } +exec_mode_error() { + echo "ERROR: --profile and --dependency options are mutually exclusive" + usage_short + exit 1 +} + list_versions() { if [[ ! -d "${PROFILES_DIR}" ]]; then @@ -274,6 +282,9 @@ while [[ $# -gt 0 ]]; do case ${key} in -p | --profile) + + [ -n "${EXECUTION_MODE}" ] && exec_mode_error || EXECUTION_MODE='profile' + if [[ -z "$2" ]]; then echo "ERROR: Missing argument for -p/--profile option" exit 1 @@ -292,6 +303,8 @@ while [[ $# -gt 0 ]]; do ;; -d | --dependency) + [ -n "${EXECUTION_MODE}" ] && exec_mode_error || EXECUTION_MODE='dependency' + if [[ -z "$2" ]]; then echo "ERROR: Missing argument for -d/--dependency option" exit diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index cbc6a7fb0..659271685 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -32,6 +32,8 @@ COMMON_GIT_FLAGS="--quiet --single-branch -c advice.detachedHead=false" PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PATCH_DIR="${PATCH_DIR}/patches" DEPENDENCY="" + +EXECUTION_MODE= VERBOSE=false DEFAULT_PROFILE="default" @@ -289,10 +291,9 @@ wgetdeps() { usage_short() { echo " -usage: dl_dep.sh [-h] - [-l [[PROFILE_NAME:]VERSION]] - [-p PROFILE_NAME[:VERSION]] - [-d DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]] +usage: dl_dep.sh [ -p PROFILE_NAME[:PROFILE_VERSION] | + -d DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] ] + [ -l [[PROFILE_NAME:]PROFILE_VERSION] ] [ -h ] DESTINATION_PATH " } @@ -309,26 +310,32 @@ positional arguments: optional arguments: -h, --help shows this help message and exits - -l, --list-dependencies [[PROFILE_NAME:]VERSION] + -l, --list-dependencies [[PROFILE_NAME:]PROFILE_VERSION] list dependency configuration profiles available for download - -p, --profile PROFILE_NAME[:VERSION] + -p, --profile PROFILE_NAME[:PROFILE_VERSION] allows downloading a pre-defined set of dependencies as defined - in ${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to + in \${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to deploy specific library versions and/or configurations, using a recognizable name. Optionally, PROFILE_NAME may include a specific version for the profile, e.g. 'mogon2:latest' or 'ngio:0.8.0', which will download the dependencies defined for that specific version. If unspecified, the 'default:latest' profile will be used, which should include all the possible dependencies. - -d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:VERSION]] + -d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] build and install a specific dependency, ignoring any --profile option provided. If PROFILE_NAME is unspecified, the 'default' - profile will be used. Similarly, if VERSION is unspecified, the - 'latest' version of the specified profile will be used. + profile will be used. Similarly, if PROFILE_VERSION is unspecified, + the 'latest' version of the specified profile will be used. -v, --verbose Increase download verbosity " } +exec_mode_error() { + echo "ERROR: --profile and --dependency options are mutually exclusive" + usage_short + exit 1 +} + # load default profile for now, might be overridden later load_profile "${DEFAULT_PROFILE}" "${DEFAULT_VERSION}" @@ -341,6 +348,9 @@ while [[ $# -gt 0 ]]; do case ${key} in -p | --profile) + + [ -n "${EXECUTION_MODE}" ] && exec_mode_error || EXECUTION_MODE='profile' + if [[ -z "$2" ]]; then echo "ERROR: Missing argument for -p/--profile option" exit 1 @@ -360,6 +370,9 @@ while [[ $# -gt 0 ]]; do ;; -d | --dependency) + + [ -n "${EXECUTION_MODE}" ] && exec_mode_error || EXECUTION_MODE='dependency' + if [[ -z "$2" ]]; then echo "ERROR: Missing argument for -d/--dependency option" exit 1 -- GitLab From 15d7f2ae305ab4769e66fc765c627752a8b1c23e Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 4 Nov 2021 12:44:17 +0100 Subject: [PATCH 03/19] dep_scripts: Rename dep to dep_name --- scripts/compile_dep.sh | 10 +++++----- scripts/dl_dep.sh | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index e52369f20..7eb8b85dd 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -427,20 +427,20 @@ if [[ -n "${DEPENDENCY}" && ! -n "${PROFILE_DEP_NAMES[${DEPENDENCY}]}" ]]; then exit fi -for dep in "${PROFILE_DEP_LIST[@]}"; do +for dep_name in "${PROFILE_DEP_LIST[@]}"; do - if [[ -n "${DEPENDENCY}" && "${dep}" != "${DEPENDENCY}" ]]; then + if [[ -n "${DEPENDENCY}" && "${dep_name}" != "${DEPENDENCY}" ]]; then continue fi - install_script="${PROFILES_DIR}/${PROFILE_VERSION}/install/${dep}.install" + install_script="${PROFILES_DIR}/${PROFILE_VERSION}/install/${dep_name}.install" - echo -e "\n\n######## Installing: ${dep} ###############################\n" + echo -e "\n\n######## Installing: ${dep_name} ###############################\n" if [[ -f "${install_script}" ]]; then source "${install_script}" else - echo "WARNING: Install script for '${dep}' not found. Skipping." + echo "WARNING: Install script for '${dep_name}' not found. Skipping." continue fi diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 659271685..e91add414 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -421,20 +421,20 @@ echo "------------------------------------" mkdir -p "${SOURCE_DIR}" ## download dependencies -for dep in "${PROFILE_DEP_NAMES[@]}"; do +for dep_name in "${PROFILE_DEP_NAMES[@]}"; do - if [[ -n "${DEPENDENCY}" && "${dep}" != "${DEPENDENCY}" ]]; then + if [[ -n "${DEPENDENCY}" && "${dep_name}" != "${DEPENDENCY}" ]]; then continue fi - if [[ ! -z "${PROFILE_WGETDEPS[${dep}]:-}" ]]; then + if [[ ! -z "${PROFILE_WGETDEPS[${dep_name}]:-}" ]]; then # dependency names can include a TAG after a colon (e.g. ofi:verbs), # remove it - dep_id=${dep%%:*} + dep_id=${dep_name%%:*} # find required version for dependency - dep_version="${PROFILE_WGETDEPS[${dep}]}" + dep_version="${PROFILE_WGETDEPS[${dep_name}]}" # build URL for dependency dep_url="${PROFILE_SOURCES[${dep_id}]}" @@ -448,16 +448,16 @@ for dep in "${PROFILE_DEP_NAMES[@]}"; do wgetdeps "${dep_id}" "${dep_url}" & - elif [[ ! -z "${PROFILE_CLONEDEPS[${dep}]:-}" ]]; then + elif [[ ! -z "${PROFILE_CLONEDEPS[${dep_name}]:-}" ]]; then # dependency names can include a TAG after a colon (e.g. ofi:verbs), # remove it - dep_id=${dep%%:*} + dep_id=${dep_name%%:*} dep_args="" # find required version for dependency - dep_version="${PROFILE_CLONEDEPS[${dep}]}" + dep_version="${PROFILE_CLONEDEPS[${dep_name}]}" # version may be a commit hash, a tag or something like HEAD@BRANCH_NAME # if it's the latter, remove the @BRANCH_NAME @@ -477,14 +477,14 @@ for dep in "${PROFILE_DEP_NAMES[@]}"; do dep_url="${dep_url/\{\{VERSION\}\}/${dep_version}}" # check if extra args are required - dep_args+="${PROFILE_CLONEDEPS_ARGS[${dep}]}" + dep_args+="${PROFILE_CLONEDEPS_ARGS[${dep_name}]}" - dep_patch=${PROFILE_CLONEDEPS_PATCHES[${dep}]} + dep_patch=${PROFILE_CLONEDEPS_PATCHES[${dep_name}]} - clonedeps "${dep}" "${dep_url}" "${dep_version}" "${dep_args}" "${dep_patch}" & + clonedeps "${dep_name}" "${dep_url}" "${dep_version}" "${dep_args}" "${dep_patch}" & else - echo "Unknown dependency '${dep}'." + echo "Unknown dependency '${dep_name}'." exit 1 fi done -- GitLab From fb45f296e4f928a0ac23900749dbeba7674d4345 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 5 Nov 2021 10:18:13 +0100 Subject: [PATCH 04/19] dep_scripts: Add 'bats' submodules for script testing --- .gitmodules | 12 ++++++++++++ tests/scripts/bats | 1 + tests/scripts/helpers/bats-assert | 1 + tests/scripts/helpers/bats-file | 1 + tests/scripts/helpers/bats-support | 1 + 5 files changed, 16 insertions(+) create mode 160000 tests/scripts/bats create mode 160000 tests/scripts/helpers/bats-assert create mode 160000 tests/scripts/helpers/bats-file create mode 160000 tests/scripts/helpers/bats-support diff --git a/.gitmodules b/.gitmodules index 30e601bf2..bb7c8ebc3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,15 @@ [submodule "external/fmt"] path = external/fmt url = https://github.com/fmtlib/fmt +[submodule "tests/scripts/bats"] + path = tests/scripts/bats + url = https://github.com/bats-core/bats-core.git +[submodule "tests/scripts/helpers/bats-assert"] + path = tests/scripts/helpers/bats-assert + url = https://github.com/bats-core/bats-assert.git +[submodule "tests/scripts/helpers/bats-support"] + path = tests/scripts/helpers/bats-support + url = https://github.com/bats-core/bats-support.git +[submodule "tests/scripts/helpers/bats-file"] + path = tests/scripts/helpers/bats-file + url = https://github.com/bats-core/bats-file.git diff --git a/tests/scripts/bats b/tests/scripts/bats new file mode 160000 index 000000000..a8b8d0dc7 --- /dev/null +++ b/tests/scripts/bats @@ -0,0 +1 @@ +Subproject commit a8b8d0dc7a05abe10c8511fe5041c137e06d18bf diff --git a/tests/scripts/helpers/bats-assert b/tests/scripts/helpers/bats-assert new file mode 160000 index 000000000..672ad1823 --- /dev/null +++ b/tests/scripts/helpers/bats-assert @@ -0,0 +1 @@ +Subproject commit 672ad1823a4d2f0c475fdbec0c4497498eec5f41 diff --git a/tests/scripts/helpers/bats-file b/tests/scripts/helpers/bats-file new file mode 160000 index 000000000..17fa557f6 --- /dev/null +++ b/tests/scripts/helpers/bats-file @@ -0,0 +1 @@ +Subproject commit 17fa557f6fe28a327933e3fa32efef1d211caa5a diff --git a/tests/scripts/helpers/bats-support b/tests/scripts/helpers/bats-support new file mode 160000 index 000000000..d140a6504 --- /dev/null +++ b/tests/scripts/helpers/bats-support @@ -0,0 +1 @@ +Subproject commit d140a65044b2d6810381935ae7f0c94c7023c8c3 -- GitLab From 641f2e1084da182ac7bd9ea6d3378691bf398a89 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 5 Nov 2021 10:28:01 +0100 Subject: [PATCH 05/19] dep_scripts: Add --dry-run option --- scripts/compile_dep.sh | 25 +++++++++++++++------- scripts/dl_dep.sh | 47 +++++++++++++++++++++++++----------------- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 7eb8b85dd..c2e5bb89b 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -36,6 +36,7 @@ INSTALL_DIR="" PERFORM_TEST= EXECUTION_MODE= +DRY_RUN=false DEFAULT_PROFILE="default" DEFAULT_VERSION="latest" @@ -49,10 +50,10 @@ declare -A PROFILE_CLONEDEPS_ARGS PROFILE_CLONEDEPS_PATCHES usage_short() { echo " -usage: compile_dep.sh [ -p PROFILE_NAME[:PROFILE_VERSION] | - -d DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] ] - [ -l [PROFILE_NAME:[PROFILE_VERSION]] ] - [ -j COMPILE_CORES] [ -h ] +usage: compile_dep.sh -p PROFILE_NAME[:PROFILE_VERSION] | + -d DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] | + -l [PROFILE_NAME:[PROFILE_VERSION]] | -n | -h + [ -j COMPILE_CORES] SOURCES_PATH INSTALL_PATH " } @@ -70,7 +71,7 @@ positional arguments: optional arguments: -h, --help shows this help message and exits - -l, --list-dependencies + -l, --list-dependencies [[PROFILE_NAME:]PROFILE_VERSION] list dependencies available for building and installation -p, --profile PROFILE_NAME[:PROFILE_VERSION] allows installing a pre-defined set of dependencies as defined @@ -91,6 +92,8 @@ optional arguments: number of cores that are used to compile the dependencies defaults to number of available cores -t, --test Perform libraries tests. + -n, --dry-run + Do not actually run, print only what would be done. " } @@ -371,6 +374,10 @@ while [[ $# -gt 0 ]]; do exit #shift # past argument ;; + -n | --dry-run) + DRY_RUN=true + shift + ;; *) # unknown option POSITIONAL+=("$1") # save it in an array for later shift # past argument @@ -438,15 +445,17 @@ for dep_name in "${PROFILE_DEP_LIST[@]}"; do echo -e "\n\n######## Installing: ${dep_name} ###############################\n" if [[ -f "${install_script}" ]]; then - source "${install_script}" + [[ "$DRY_RUN" == true ]] || source "${install_script}" else echo "WARNING: Install script for '${dep_name}' not found. Skipping." continue fi - pkg_install + if [[ "$DRY_RUN" == false ]]; then + pkg_install - [ "${PERFORM_TEST}" ] && pkg_check + [ "${PERFORM_TEST}" ] && pkg_check + fi done diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index e91add414..7218def7b 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -35,6 +35,7 @@ DEPENDENCY="" EXECUTION_MODE= VERBOSE=false +DRY_RUN=false DEFAULT_PROFILE="default" DEFAULT_VERSION="latest" @@ -247,19 +248,19 @@ clonedeps() { local ACTION if [[ -d "${SOURCE_DIR}/${FOLDER}/.git" ]]; then - cd "${SOURCE_DIR}/${FOLDER}" && git fetch -q + [[ "$DRY_RUN" == true ]] || (cd "${SOURCE_DIR}/${FOLDER}" && git fetch -q) ACTION="Pulled" else - git clone ${COMMON_GIT_FLAGS} ${GIT_FLAGS} -- "${REPO}" "${SOURCE_DIR}/${FOLDER}" + [[ "$DRY_RUN" == true ]] || (git clone ${COMMON_GIT_FLAGS} ${GIT_FLAGS} -- "${REPO}" "${SOURCE_DIR}/${FOLDER}") ACTION="Cloned" fi # fix the version - cd "${SOURCE_DIR}/${FOLDER}" && git checkout -qf "${COMMIT}" + [[ "$DRY_RUN" == true ]] || (cd "${SOURCE_DIR}/${FOLDER}" && git checkout -qf "${COMMIT}") echo "${ACTION} '${REPO}' to '${FOLDER}' with commit '[${COMMIT}]' and flags '${GIT_FLAGS}'" # apply patch if provided if [[ -n "${PATCH}" ]]; then - git apply --verbose "${PATCH_DIR}/${PATCH}" + [[ "$DRY_RUN" == true ]] || git apply --verbose "${PATCH_DIR}/${PATCH}" fi } @@ -273,27 +274,30 @@ wgetdeps() { FOLDER=$1 URL=$2 - if [[ -d "${SOURCE_DIR}/${FOLDER}" ]]; then - # SC2115 Use "${var:?}" to ensure this never expands to /* . - rm -rf "${SOURCE_DIR:?}/${FOLDER:?}" - fi - mkdir -p "${SOURCE_DIR}/${FOLDER}" - cd "${SOURCE_DIR}" - FILENAME="$(basename $URL)" - if [[ -f "${SOURCE_DIR}/$FILENAME" ]]; then - rm -f "${SOURCE_DIR}/$FILENAME" + + if [[ "$DRY_RUN" == false ]]; then + if [[ -d "${SOURCE_DIR}/${FOLDER}" ]]; then + # SC2115 Use "${var:?}" to ensure this never expands to /* . + rm -rf "${SOURCE_DIR:?}/${FOLDER:?}" + fi + mkdir -p "${SOURCE_DIR}/${FOLDER}" + cd "${SOURCE_DIR}" + FILENAME="$(basename $URL)" + if [[ -f "${SOURCE_DIR}/$FILENAME" ]]; then + rm -f "${SOURCE_DIR}/$FILENAME" + fi + curl ${COMMON_CURL_FLAGS} "$URL" || error_exit "Failed to download ${URL}" $? + tar -xf "$FILENAME" --directory "${SOURCE_DIR}/${FOLDER}" --strip-components=1 + rm -f "$FILENAME" fi - curl ${COMMON_CURL_FLAGS} "$URL" || error_exit "Failed to download ${URL}" $? - tar -xf "$FILENAME" --directory "${SOURCE_DIR}/${FOLDER}" --strip-components=1 - rm -f "$FILENAME" echo "Downloaded '${URL}' to '${FOLDER}'" } usage_short() { echo " -usage: dl_dep.sh [ -p PROFILE_NAME[:PROFILE_VERSION] | - -d DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] ] - [ -l [[PROFILE_NAME:]PROFILE_VERSION] ] [ -h ] +usage: dl_dep.sh -p PROFILE_NAME[:PROFILE_VERSION] | + -d DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] | + -l [[PROFILE_NAME:]PROFILE_VERSION] | -n | -h DESTINATION_PATH " } @@ -326,6 +330,7 @@ optional arguments: option provided. If PROFILE_NAME is unspecified, the 'default' profile will be used. Similarly, if PROFILE_VERSION is unspecified, the 'latest' version of the specified profile will be used. + -n, --dry-run Do not actually run, print only what would be done. -v, --verbose Increase download verbosity " } @@ -399,6 +404,10 @@ while [[ $# -gt 0 ]]; do VERBOSE=true shift # past argument ;; + -n | --dry-run) + DRY_RUN=true + shift + ;; *) # unknown option POSITIONAL+=("$1") # save it in an array for later shift # past argument -- GitLab From 088dfb5c9c65885b49ce1b9f5f1aeb8d2405a4d5 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 5 Nov 2021 10:33:39 +0100 Subject: [PATCH 06/19] dep_scripts: Fix issues with explicit profile names in dl_dep.sh --- scripts/compile_dep.sh | 3 ++- scripts/dl_dep.sh | 56 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index c2e5bb89b..cc5c4e667 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -431,11 +431,12 @@ export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH}" if [[ -n "${DEPENDENCY}" && ! -n "${PROFILE_DEP_NAMES[${DEPENDENCY}]}" ]]; then echo "Dependency '${DEPENDENCY}' not found in '${PROFILE_NAME}:${PROFILE_VERSION}'" - exit + exit 1 fi for dep_name in "${PROFILE_DEP_LIST[@]}"; do + # in dependency mode, skip any dependencies != DEPENDENCY if [[ -n "${DEPENDENCY}" && "${dep_name}" != "${DEPENDENCY}" ]]; then continue fi diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 7218def7b..f24653ba1 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -41,7 +41,8 @@ DEFAULT_PROFILE="default" DEFAULT_VERSION="latest" PROFILES_DIR="${PWD}/profiles" SOURCES_FILE="${PROFILES_DIR}/sources.list" -declare -a PROFILE_DEP_NAMES +declare -a PROFILE_DEP_LIST +declare -A PROFILE_DEP_NAMES declare -A PROFILE_WGETDEPS PROFILE_CLONEDEPS PROFILE_SOURCES declare -A PROFILE_CLONEDEPS_ARGS PROFILE_CLONEDEPS_PATCHES @@ -164,6 +165,7 @@ load_profile() { # make sure we are in a known state PROFILE_DEP_NAMES=() + PROFILE_DEP_LIST=() PROFILE_CLONEDEPS=() PROFILE_CLONEDEPS_ARGS=() PROFILE_CLONEDEPS_PATCHES=() @@ -196,7 +198,8 @@ load_profile() { # propagate results outside of function for i in "${!order[@]}"; do - PROFILE_DEP_NAMES[$i]="${order[${i}]}" + PROFILE_DEP_LIST[$i]="${order[${i}]}" + PROFILE_DEP_NAMES["${order[$i]}"]="$i" done for k in "${!clonedeps[@]}"; do @@ -369,7 +372,6 @@ while [[ $# -gt 0 ]]; do PROFILE_VERSION="${DEFAULT_VERSION}" fi - load_profile "${PROFILE_NAME}" "${PROFILE_VERSION}" shift # past argument shift # past value ;; @@ -382,7 +384,43 @@ while [[ $# -gt 0 ]]; do echo "ERROR: Missing argument for -d/--dependency option" exit 1 fi - DEPENDENCY="$2" + + PROFILE_NAME=${DEFAULT_PROFILE} + PROFILE_VERSION=${DEFAULT_VERSION} + + # e.g. mercury@mogon1:latest + if [[ "$2" =~ ^(.*)@(.*):(.*)$ ]]; then + if [[ -n "${BASH_REMATCH[1]}" ]]; then + DEPENDENCY="${BASH_REMATCH[1]}" + fi + + if [[ -n "${BASH_REMATCH[2]}" ]]; then + PROFILE_NAME="${BASH_REMATCH[2]}" + fi + + if [[ -n "${BASH_REMATCH[3]}" ]]; then + PROFILE_VERSION="${BASH_REMATCH[3]}" + fi + + # e.g. mercury@mogon1 + elif [[ "$2" =~ ^(.*)@(.*)$ ]]; then + if [[ -n "${BASH_REMATCH[1]}" ]]; then + DEPENDENCY="${BASH_REMATCH[1]}" + fi + + if [[ -n "${BASH_REMATCH[2]}" ]]; then + PROFILE_NAME="${BASH_REMATCH[2]}" + fi + # e.g. mercury + else + DEPENDENCY="$2" + fi + + if [[ ! -n "${DEPENDENCY}" ]]; then + echo "ERROR: Missing dependency name." + exit 1 + fi + shift # past argument shift # past value ;; @@ -424,14 +462,22 @@ if [[ -z ${1+x} ]]; then fi SOURCE_DIR="$(readlink -mn "${1}")" +load_profile "${PROFILE_NAME}" "${PROFILE_VERSION}" + +if [[ -n "${DEPENDENCY}" && ! -n "${PROFILE_DEP_NAMES[${DEPENDENCY}]}" ]]; then + echo "ERROR: '${DEPENDENCY}' not found in '${PROFILE_NAME}:${PROFILE_VERSION}'" + exit 1 +fi + echo "Destination path is set to \"${SOURCE_DIR}\"" echo "------------------------------------" mkdir -p "${SOURCE_DIR}" ## download dependencies -for dep_name in "${PROFILE_DEP_NAMES[@]}"; do +for dep_name in "${PROFILE_DEP_LIST[@]}"; do + # in dependency mode, skip any dependencies != DEPENDENCY if [[ -n "${DEPENDENCY}" && "${dep_name}" != "${DEPENDENCY}" ]]; then continue fi -- GitLab From 46776074ebbb0e5f429c9543f9a577e32c46d898 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Sun, 7 Nov 2021 18:17:46 +0100 Subject: [PATCH 07/19] dep_scripts: Increase verbosity --- scripts/compile_dep.sh | 4 +++- scripts/dl_dep.sh | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index cc5c4e667..236a02b0a 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -418,9 +418,11 @@ CMAKE="${CMAKE} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}" echo "Sources download path = ${SOURCE_DIR}" echo "Installation path = ${INSTALL_DIR}" +echo "Profile name: ${PROFILE_NAME}" +echo "Profile version: ${PROFILE_VERSION}" echo "------------------------------------" -mkdir -p "${SOURCE_DIR}" +mkdir -p "${SOURCE_DIR}" || exit 1 ######### From now on exits on any error ######## set -e diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index f24653ba1..5ef23f1a9 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -470,9 +470,11 @@ if [[ -n "${DEPENDENCY}" && ! -n "${PROFILE_DEP_NAMES[${DEPENDENCY}]}" ]]; then fi echo "Destination path is set to \"${SOURCE_DIR}\"" +echo "Profile name: ${PROFILE_NAME}" +echo "Profile version: ${PROFILE_VERSION}" echo "------------------------------------" -mkdir -p "${SOURCE_DIR}" +mkdir -p "${SOURCE_DIR}" || exit 1 ## download dependencies for dep_name in "${PROFILE_DEP_LIST[@]}"; do -- GitLab From af2dc5d8750af825076b5e1501c96a1944b9fc45 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Sun, 7 Nov 2021 18:34:14 +0100 Subject: [PATCH 08/19] dep_scripts: Add tests for dl_dep.sh and compile_dep.sh --- .gitignore | 4 +- tests/scripts/compile_dep.sh/0.8.0/all.out | 14 ++ tests/scripts/compile_dep.sh/0.8.0/ci.out | 8 + .../scripts/compile_dep.sh/0.8.0/default.out | 18 ++ tests/scripts/compile_dep.sh/0.8.0/direct.out | 6 + tests/scripts/compile_dep.sh/0.8.0/mogon1.out | 11 + tests/scripts/compile_dep.sh/0.8.0/mogon2.out | 15 ++ tests/scripts/compile_dep.sh/0.8.0/ngio.out | 14 ++ tests/scripts/compile_dep.sh/latest | 1 + tests/scripts/compile_dep.sh/test.bats | 203 ++++++++++++++++++ tests/scripts/dl_dep.sh/0.8.0/all.out | 14 ++ tests/scripts/dl_dep.sh/0.8.0/ci.out | 8 + tests/scripts/dl_dep.sh/0.8.0/default.out | 16 ++ tests/scripts/dl_dep.sh/0.8.0/direct.out | 6 + tests/scripts/dl_dep.sh/0.8.0/mogon1.out | 11 + tests/scripts/dl_dep.sh/0.8.0/mogon2.out | 13 ++ tests/scripts/dl_dep.sh/0.8.0/ngio.out | 13 ++ tests/scripts/dl_dep.sh/latest | 1 + tests/scripts/dl_dep.sh/test.bats | 188 ++++++++++++++++ 19 files changed, 562 insertions(+), 2 deletions(-) create mode 100644 tests/scripts/compile_dep.sh/0.8.0/all.out create mode 100644 tests/scripts/compile_dep.sh/0.8.0/ci.out create mode 100644 tests/scripts/compile_dep.sh/0.8.0/default.out create mode 100644 tests/scripts/compile_dep.sh/0.8.0/direct.out create mode 100644 tests/scripts/compile_dep.sh/0.8.0/mogon1.out create mode 100644 tests/scripts/compile_dep.sh/0.8.0/mogon2.out create mode 100644 tests/scripts/compile_dep.sh/0.8.0/ngio.out create mode 120000 tests/scripts/compile_dep.sh/latest create mode 100644 tests/scripts/compile_dep.sh/test.bats create mode 100644 tests/scripts/dl_dep.sh/0.8.0/all.out create mode 100644 tests/scripts/dl_dep.sh/0.8.0/ci.out create mode 100644 tests/scripts/dl_dep.sh/0.8.0/default.out create mode 100644 tests/scripts/dl_dep.sh/0.8.0/direct.out create mode 100644 tests/scripts/dl_dep.sh/0.8.0/mogon1.out create mode 100644 tests/scripts/dl_dep.sh/0.8.0/mogon2.out create mode 100644 tests/scripts/dl_dep.sh/0.8.0/ngio.out create mode 120000 tests/scripts/dl_dep.sh/latest create mode 100644 tests/scripts/dl_dep.sh/test.bats diff --git a/.gitignore b/.gitignore index ec58526e1..a6a9e0756 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,7 @@ # Executables *.exe -*.out +#*.out *.app # ---> C @@ -53,7 +53,7 @@ # Executables *.exe -*.out +#*.out *.app *.i*86 *.x86_64 diff --git a/tests/scripts/compile_dep.sh/0.8.0/all.out b/tests/scripts/compile_dep.sh/0.8.0/all.out new file mode 100644 index 000000000..847a6c563 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.8.0/all.out @@ -0,0 +1,14 @@ +######## Installing: bzip2 ############################### +######## Installing: zstd ############################### +######## Installing: lz4 ############################### +######## Installing: snappy ############################### +######## Installing: capstone ############################### +######## Installing: bmi ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### diff --git a/tests/scripts/compile_dep.sh/0.8.0/ci.out b/tests/scripts/compile_dep.sh/0.8.0/ci.out new file mode 100644 index 000000000..8c7072d87 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.8.0/ci.out @@ -0,0 +1,8 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### diff --git a/tests/scripts/compile_dep.sh/0.8.0/default.out b/tests/scripts/compile_dep.sh/0.8.0/default.out new file mode 100644 index 000000000..dfd76cf06 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.8.0/default.out @@ -0,0 +1,18 @@ +######## Installing: bzip2 ############################### +######## Installing: zstd ############################### +######## Installing: lz4 ############################### +######## Installing: snappy ############################### +######## Installing: capstone ############################### +######## Installing: bmi ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: rocksdb:experimental ############################### +WARNING: Install script for 'rocksdb:experimental' not found. Skipping. +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: agios ############################### diff --git a/tests/scripts/compile_dep.sh/0.8.0/direct.out b/tests/scripts/compile_dep.sh/0.8.0/direct.out new file mode 100644 index 000000000..d830e84df --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.8.0/direct.out @@ -0,0 +1,6 @@ +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.8.0/mogon1.out b/tests/scripts/compile_dep.sh/0.8.0/mogon1.out new file mode 100644 index 000000000..572319b9a --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.8.0/mogon1.out @@ -0,0 +1,11 @@ +######## Installing: zstd ############################### +######## Installing: lz4 ############################### +######## Installing: snappy ############################### +######## Installing: capstone ############################### +######## Installing: libfabric:verbs ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.8.0/mogon2.out b/tests/scripts/compile_dep.sh/0.8.0/mogon2.out new file mode 100644 index 000000000..b18dc3762 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.8.0/mogon2.out @@ -0,0 +1,15 @@ +######## Installing: bzip2 ############################### +######## Installing: zstd ############################### +######## Installing: lz4 ############################### +######## Installing: snappy ############################### +######## Installing: capstone ############################### +######## Installing: libfabric:experimental ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb:experimental ############################### +WARNING: Install script for 'rocksdb:experimental' not found. Skipping. +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. diff --git a/tests/scripts/compile_dep.sh/0.8.0/ngio.out b/tests/scripts/compile_dep.sh/0.8.0/ngio.out new file mode 100644 index 000000000..a0f9661d4 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.8.0/ngio.out @@ -0,0 +1,14 @@ +######## Installing: zstd ############################### +######## Installing: lz4 ############################### +######## Installing: snappy ############################### +######## Installing: capstone ############################### +######## Installing: libfabric:experimental ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: agios ############################### diff --git a/tests/scripts/compile_dep.sh/latest b/tests/scripts/compile_dep.sh/latest new file mode 120000 index 000000000..6c49bdfd1 --- /dev/null +++ b/tests/scripts/compile_dep.sh/latest @@ -0,0 +1 @@ +0.8.0/ \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/test.bats b/tests/scripts/compile_dep.sh/test.bats new file mode 100644 index 000000000..f9d3fb5eb --- /dev/null +++ b/tests/scripts/compile_dep.sh/test.bats @@ -0,0 +1,203 @@ +# vim: set ft=bash: + +setup() { + GIT_ROOTDIR=$(git rev-parse --show-toplevel) + SCRIPT_TESTDIR=${GIT_ROOTDIR}/tests/scripts/compile_dep.sh + + load "${GIT_ROOTDIR}/tests/scripts/helpers/bats-support/load.bash" + load "${GIT_ROOTDIR}/tests/scripts/helpers/bats-assert/load.bash" + load "${GIT_ROOTDIR}/tests/scripts/helpers/bats-file/load.bash" + + + # save the current PWD + pushd . > /dev/null + + # make sure that the script is able to find the profiles + cd ${GIT_ROOTDIR}/scripts + + # make executables in scripts/ visible to PATH + PATH="${GIT_ROOTDIR}/scripts:$PATH" +} + +teardown() { + # restore the current PWD + popd > /dev/null +} + +@test "[compile_dep.sh] Check usage" { + run compile_dep.sh -h + assert_output --partial "usage: compile_dep.sh" +} + +@test "[compile_dep.sh] Check --profile and --dependency option at the same time" { + run -1 compile_dep.sh -p foobar -d barbaz + assert_output --partial "ERROR: --profile and --dependency options are mutually exclusive" +} + +@test "[compile_dep.sh] Check --profile option without argument" { + run -1 compile_dep.sh -p + assert_output --partial "ERROR: Missing argument for -p/--profile option" +} + +@test "[compile_dep.sh] Check invalid --profile option without SOURCES_PATH or INSTALL_PATH" { + run -1 compile_dep.sh -p foo + assert_output --partial "ERROR: Positional arguments missing." +} + +@test "[compile_dep.sh] Check invalid --profile option without SOURCES_PATH" { + run -1 compile_dep.sh -p foo bar + assert_output --partial "ERROR: Positional arguments missing." +} + +@test "[compile_dep.sh] Check invalid --profile option" { + run -1 compile_dep.sh -p foo bar baz + assert_output --partial "Profile 'foo:latest' does not exist." +} + +@test "[compile_dep.sh] Check valid --profile option, no PROFILE_VERSION" { + + profile_names=($(compile_dep.sh -l latest | perl -lne "print for /^\* (\w+):.*$/")) + + for pn in "${profile_names[@]}"; do + + expected_output=${SCRIPT_TESTDIR}/latest/${pn}.out + + assert_exist $expected_output + + run compile_dep.sh -n -p "${pn}" foo bar + + while IFS= read -r line + do + assert_output --partial "$line" + done < "$expected_output" + done + +} + +@test "[compile_dep.sh] Check valid --profile option, with PROFILE_VERSION" { + + profile_versions=($(compile_dep.sh -l | perl -lne "print for /^\s{2}(.*?):$/")) + + for pv in "${profile_versions[@]}"; do + profile_names=($(compile_dep.sh -l ${pv} | perl -lne "print for /^\* (\w+):.*$/")) + + for pn in "${profile_names[@]}"; do + + expected_output=${SCRIPT_TESTDIR}/${pv}/${pn}.out + + assert_exist $expected_output + + run compile_dep.sh -n -p "${pn}:${pv}" foo bar + + while IFS= read -r line + do + assert_output --partial "$line" + done < "$expected_output" + done + + done +} + +@test "[compile_dep.sh] Check unknown --dependency option" { + + profile_name=default + profile_version=latest + + run -1 compile_dep.sh -n -d foobar bar baz + + assert_output --partial "Dependency 'foobar' not found in '$profile_name:$profile_version'" + +} + +@test "[compile_dep.sh] Check valid --dependency option, no PROFILE_NAME or PROFILE_VERSION" { + + dependency_names=($(compile_dep.sh -l default:latest | perl -lne "print for /^\s{4}(\w+(?::\w+)*):.*$/")) + + for dep in "${dependency_names[@]}"; do + source_path=$(readlink -f ${BATS_TEST_TMPDIR}/foo) + install_path=$(readlink -f ${BATS_TEST_TMPDIR}/bar) + + run compile_dep.sh -n -d "${dep}" ${source_path} ${install_path} + + assert_output --regexp \ +"CORES = 8 \(default\) +Sources download path = ${source_path} +Installation path = ${install_path} +Profile name: default +Profile version: latest +------------------------------------ + + +######## Installing: ${dep} ############################### +.* +Done" + + done +} + +@test "[compile_dep.sh] Check valid --dependency option, PROFILE_NAME only" { + + profile_names=($(compile_dep.sh -l latest | perl -lne "print for /^\* (\w+):.*$/")) + + for pn in "${profile_names[@]}"; do + + dependency_names=($(compile_dep.sh -l ${pn}:latest | perl -lne "print for /^\s{4}(\w+(?::\w+)*):.*$/")) + + for dep in "${dependency_names[@]}"; do + source_path=$(readlink -f ${BATS_TEST_TMPDIR}/foo) + install_path=$(readlink -f ${BATS_TEST_TMPDIR}/bar) + + run compile_dep.sh -n -d "${dep}@${pn}" ${source_path} ${install_path} + + assert_output --regexp \ +"CORES = 8 \(default\) +Sources download path = ${source_path} +Installation path = ${install_path} +Profile name: ${pn} +Profile version: latest +------------------------------------ + + +######## Installing: ${dep} ############################### +.* +Done" + + done + done +} + +@test "[compile_dep.sh] Check valid --dependency option, PROFILE_NAME and PROFILE_VERSION" { + + profile_versions=($(compile_dep.sh -l | perl -lne "print for /^\s{2}(.*?):$/")) + + for pv in "${profile_versions[@]}"; do + + profile_names=($(compile_dep.sh -l latest | perl -lne "print for /^\* (\w+):.*$/")) + + for pn in "${profile_names[@]}"; do + + dependency_names=($(compile_dep.sh -l ${pn}:latest | perl -lne "print for /^\s{4}(\w+(?::\w+)*):.*$/")) + + for dep in "${dependency_names[@]}"; do + source_path=$(readlink -f ${BATS_TEST_TMPDIR}/foo) + install_path=$(readlink -f ${BATS_TEST_TMPDIR}/bar) + + run compile_dep.sh -n -d "${dep}@${pn}" ${source_path} ${install_path} + + assert_output --regexp \ +"CORES = 8 \(default\) +Sources download path = ${source_path} +Installation path = ${install_path} +Profile name: ${pn} +Profile version: latest +------------------------------------ + + +######## Installing: ${dep} ############################### +.* +Done" + + done + done + done +} diff --git a/tests/scripts/dl_dep.sh/0.8.0/all.out b/tests/scripts/dl_dep.sh/0.8.0/all.out new file mode 100644 index 000000000..9253d8407 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.8.0/all.out @@ -0,0 +1,14 @@ +Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' +Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' +Cloned 'https://github.com/radix-io/bmi/' to 'bmi' with commit '[6ea0b78fce1b964e45102828cdd05df7040a94c8]' and flags '' +Downloaded 'https://sourceforge.net/projects/bzip2/files/bzip2-1.0.6.tar.gz' to 'bzip2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '-b v1.8.1' +Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '-b development' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.8.0/ci.out b/tests/scripts/dl_dep.sh/0.8.0/ci.out new file mode 100644 index 000000000..93c1eee29 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.8.0/ci.out @@ -0,0 +1,8 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '-b v1.8.1' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '-b development' diff --git a/tests/scripts/dl_dep.sh/0.8.0/default.out b/tests/scripts/dl_dep.sh/0.8.0/default.out new file mode 100644 index 000000000..967b9cb50 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.8.0/default.out @@ -0,0 +1,16 @@ +Downloaded 'https://sourceforge.net/projects/bzip2/files/bzip2-1.0.6.tar.gz' to 'bzip2' +Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' +Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' +Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' +Cloned 'https://github.com/radix-io/bmi/' to 'bmi' with commit '[6ea0b78fce1b964e45102828cdd05df7040a94c8]' and flags '' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '-b v1.8.1' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '-b development' diff --git a/tests/scripts/dl_dep.sh/0.8.0/direct.out b/tests/scripts/dl_dep.sh/0.8.0/direct.out new file mode 100644 index 000000000..7778dc065 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.8.0/direct.out @@ -0,0 +1,6 @@ +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' diff --git a/tests/scripts/dl_dep.sh/0.8.0/mogon1.out b/tests/scripts/dl_dep.sh/0.8.0/mogon1.out new file mode 100644 index 000000000..9a88c8313 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.8.0/mogon1.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' +Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' +Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:verbs' with commit '[HEAD]' and flags '-b v1.7.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Cloned 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' with commit '[6.2.2]' and flags '' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' diff --git a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out new file mode 100644 index 000000000..154ef6a3b --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out @@ -0,0 +1,13 @@ +Downloaded 'https://sourceforge.net/projects/bzip2/files/bzip2-1.0.6.tar.gz' to 'bzip2' +Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' +Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' +Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:experimental' with commit '[HEAD]' and flags '-b v1.9.1' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' diff --git a/tests/scripts/dl_dep.sh/0.8.0/ngio.out b/tests/scripts/dl_dep.sh/0.8.0/ngio.out new file mode 100644 index 000000000..6fd8740fc --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.8.0/ngio.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' +Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' +Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:experimental' with commit '[HEAD]' and flags '-b v1.9.1' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' +Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '-b development' diff --git a/tests/scripts/dl_dep.sh/latest b/tests/scripts/dl_dep.sh/latest new file mode 120000 index 000000000..6c49bdfd1 --- /dev/null +++ b/tests/scripts/dl_dep.sh/latest @@ -0,0 +1 @@ +0.8.0/ \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/test.bats b/tests/scripts/dl_dep.sh/test.bats new file mode 100644 index 000000000..784ac2b8b --- /dev/null +++ b/tests/scripts/dl_dep.sh/test.bats @@ -0,0 +1,188 @@ +# vim: set ft=bash: + +setup() { + GIT_ROOTDIR=$(git rev-parse --show-toplevel) + SCRIPT_TESTDIR=${GIT_ROOTDIR}/tests/scripts/dl_dep.sh + + load "${GIT_ROOTDIR}/tests/scripts/helpers/bats-support/load.bash" + load "${GIT_ROOTDIR}/tests/scripts/helpers/bats-assert/load.bash" + load "${GIT_ROOTDIR}/tests/scripts/helpers/bats-file/load.bash" + + + # save the current PWD + pushd . > /dev/null + + # make sure that the script is able to find the profiles + cd ${GIT_ROOTDIR}/scripts + + # make executables in scripts/ visible to PATH + PATH="${GIT_ROOTDIR}/scripts:$PATH" +} + +teardown() { + # restore the current PWD + popd > /dev/null +} + +@test "[dl_dep.sh] Check usage" { + run dl_dep.sh -h + assert_output --partial "usage: dl_dep.sh" +} + +@test "[dl_dep.sh] Check --profile and --dependency option at the same time" { + run -1 dl_dep.sh -p foobar -d barbaz + assert_output --partial "ERROR: --profile and --dependency options are mutually exclusive" +} + +@test "[dl_dep.sh] Check --profile option without argument" { + run -1 dl_dep.sh -p + assert_output --partial "ERROR: Missing argument for -p/--profile option" +} + +@test "[dl_dep.sh] Check invalid --profile option without DESTINATION_PATH" { + run -1 dl_dep.sh -p foobar + assert_output --partial "ERROR: Positional arguments missing." +} + +@test "[dl_dep.sh] Check valid --profile option without DESTINATION_PATH" { + run dl_dep.sh -p default + assert_output --partial "ERROR: Positional arguments missing." +} + +@test "[dl_dep.sh] Check valid --profile option, no PROFILE_VERSION" { + + profile_names=($(dl_dep.sh -l latest | perl -lne "print for /^\* (\w+):.*$/")) + + for pn in "${profile_names[@]}"; do + + expected_output=${SCRIPT_TESTDIR}/latest/${pn}.out + + assert_exist $expected_output + + run dl_dep.sh -n -p "${pn}" "${BATS_TEST_TMPDIR}" + + while IFS= read -r line + do + assert_output --partial "$line" + done < "$expected_output" + done +} + +@test "[dl_dep.sh] Check valid --profile option, with PROFILE_VERSION" { + + profile_versions=($(dl_dep.sh -l | perl -lne "print for /^\s{2}(.*?):$/")) + + for pv in "${profile_versions[@]}"; do + profile_names=($(dl_dep.sh -l ${pv} | perl -lne "print for /^\* (\w+):.*$/")) + + for pn in "${profile_names[@]}"; do + + expected_output=${SCRIPT_TESTDIR}/${pv}/${pn}.out + + assert_exist $expected_output + + run dl_dep.sh -n -p "${pn}:${pv}" "${BATS_TEST_TMPDIR}" + + while IFS= read -r line + do + assert_output --partial "$line" + done < "$expected_output" + done + + done +} + +@test "[dl_dep.sh] Check unknown --dependency option" { + + profile_name=default + profile_version=latest + + run -1 dl_dep.sh -n -d foobar "${BATS_TEST_TMPDIR}" + + assert_output "ERROR: 'foobar' not found in '$profile_name:$profile_version'" + +} + +@test "[dl_dep.sh] Check invalid --dependency option without DESTINATION_PATH" { + run -1 dl_dep.sh -d foobar + assert_output --partial "ERROR: Positional arguments missing." +} + +@test "[dl_dep.sh] Check valid --dependency option without DESTINATION_PATH" { + run dl_dep.sh -d mercury + assert_output --partial "ERROR: Positional arguments missing." +} + +@test "[dl_dep.sh] Check valid --dependency option, no PROFILE_NAME or PROFILE_VERSION" { + + dependency_names=($(dl_dep.sh -l default:latest | perl -lne "print for /^\s{4}(\w+(?::\w+)*):.*$/")) + + for dep in "${dependency_names[@]}"; do + destination_path=$(readlink -f ${BATS_TEST_TMPDIR}) + + run dl_dep.sh -n -d "${dep}" "${BATS_TEST_TMPDIR}" + + assert_output --regexp \ +"Destination path is set to \"${destination_path}\" +Profile name: default +Profile version: latest +------------------------------------ +Downloaded|Cloned '.*' to '${dep}'.* +Done" + + done +} + +@test "[dl_dep.sh] Check valid --dependency option, PROFILE_NAME only" { + + profile_names=($(dl_dep.sh -l latest | perl -lne "print for /^\* (\w+):.*$/")) + + for pn in "${profile_names[@]}"; do + + dependency_names=($(dl_dep.sh -l ${pn}:latest | perl -lne "print for /^\s{4}(\w+(?::\w+)*):.*$/")) + + for dep in "${dependency_names[@]}"; do + destination_path=$(readlink -f ${BATS_TEST_TMPDIR}) + + run dl_dep.sh -n -d "${dep}@${pn}" "${BATS_TEST_TMPDIR}" + + assert_output --regexp \ +"Destination path is set to \"${destination_path}\" +Profile name: ${pn} +Profile version: latest +------------------------------------ +Downloaded|Cloned '.*' to '${dep}'.* +Done" + + done + done +} + +@test "[dl_dep.sh] Check valid --dependency option, PROFILE_NAME and PROFILE_VERSION" { + + profile_versions=($(dl_dep.sh -l | perl -lne "print for /^\s{2}(.*?):$/")) + + for pv in "${profile_versions[@]}"; do + profile_names=($(dl_dep.sh -l "${pv}" | perl -lne "print for /^\* (\w+):.*$/")) + + for pn in "${profile_names[@]}"; do + + dependency_names=($(dl_dep.sh -l ${pn}:${pv} | perl -lne "print for /^\s{4}(\w+(?::\w+)*):.*$/")) + + for dep in "${dependency_names[@]}"; do + destination_path=$(readlink -f ${BATS_TEST_TMPDIR}) + + run dl_dep.sh -n -d "${dep}@${pn}:${pv}" "${BATS_TEST_TMPDIR}" + + assert_output --regexp \ +"Destination path is set to \"${destination_path}\" +Profile name: ${pn} +Profile version: ${pv} +------------------------------------ +Downloaded|Cloned '.*' to '${dep}'.* +Done" + + done + done + done +} -- GitLab From d4c1245186260750f43ed72c073c7247de031348 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 8 Nov 2021 12:29:40 +0100 Subject: [PATCH 09/19] dep_scripts: Enable scripts testing in .gitlab-ci.yml --- .gitlab-ci.yml | 19 +++++++++++++++++++ tests/scripts/compile_dep.sh/test.bats | 6 +++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dab8f7edd..93d64cc3f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,6 +12,7 @@ variables: INTEGRATION_TESTS_BIN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration" COVERAGE_PATH: "${CI_PROJECT_DIR}/gkfs/build/.coverage" PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test" + BATS: "${CI_PROJECT_DIR}/tests/scripts/bats/bin/bats" LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/deps/install/lib:${CI_PROJECT_DIR}/deps/install/lib64" # Configuration variables GKFS_LOG_LEVEL: "100" @@ -97,6 +98,24 @@ gkfwd: ## Testing ################################################################################ +## == tests for scripts ==================== +scripts: + stage: test + image: gekkofs/testing:0.8.0 + needs: [] + parallel: + matrix: + - SUBTEST: [ dl_dep.sh, compile_dep.sh ] + script: + - mkdir -p ${BUILD_PATH}/tests/scripts + - cd ${BUILD_PATH}/tests/scripts + - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/${SUBTEST} --formatter junit > report.xml + artifacts: + expire_in: 1 week + reports: + junit: ${BUILD_PATH}/tests/scripts/report.xml + + ## == integration tests for gkfs =========== gkfs:integration: stage: test diff --git a/tests/scripts/compile_dep.sh/test.bats b/tests/scripts/compile_dep.sh/test.bats index f9d3fb5eb..ca994bc44 100644 --- a/tests/scripts/compile_dep.sh/test.bats +++ b/tests/scripts/compile_dep.sh/test.bats @@ -120,7 +120,7 @@ teardown() { run compile_dep.sh -n -d "${dep}" ${source_path} ${install_path} assert_output --regexp \ -"CORES = 8 \(default\) +"CORES = [0-9]+ \(default\) Sources download path = ${source_path} Installation path = ${install_path} Profile name: default @@ -150,7 +150,7 @@ Done" run compile_dep.sh -n -d "${dep}@${pn}" ${source_path} ${install_path} assert_output --regexp \ -"CORES = 8 \(default\) +"CORES = [0-9]+ \(default\) Sources download path = ${source_path} Installation path = ${install_path} Profile name: ${pn} @@ -185,7 +185,7 @@ Done" run compile_dep.sh -n -d "${dep}@${pn}" ${source_path} ${install_path} assert_output --regexp \ -"CORES = 8 \(default\) +"CORES = [0-9]+ \(default\) Sources download path = ${source_path} Installation path = ${install_path} Profile name: ${pn} -- GitLab From 57a502ea19b1e67364309b54575b15db307d3f22 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 8 Nov 2021 15:46:49 +0100 Subject: [PATCH 10/19] dep_scripts: Allow setting the directory for profiles --- scripts/compile_dep.sh | 34 +++++++++++++++++++++++++--------- scripts/dl_dep.sh | 34 +++++++++++++++++++++++++--------- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 236a02b0a..a90df71e2 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -50,10 +50,12 @@ declare -A PROFILE_CLONEDEPS_ARGS PROFILE_CLONEDEPS_PATCHES usage_short() { echo " -usage: compile_dep.sh -p PROFILE_NAME[:PROFILE_VERSION] | +usage: compile_dep.sh -h | + -p PROFILE_NAME[:PROFILE_VERSION] | -d DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] | - -l [PROFILE_NAME:[PROFILE_VERSION]] | -n | -h - [ -j COMPILE_CORES] + -l [PROFILE_NAME:[PROFILE_VERSION]] | + -h + [ -P PROFILES_DIR ] [ -j COMPILE_CORES] [ -n ] SOURCES_PATH INSTALL_PATH " } @@ -70,11 +72,11 @@ positional arguments: optional arguments: - -h, --help shows this help message and exits + -h, --help Shows this help message and exits -l, --list-dependencies [[PROFILE_NAME:]PROFILE_VERSION] - list dependencies available for building and installation + List dependencies available for building and installation -p, --profile PROFILE_NAME[:PROFILE_VERSION] - allows installing a pre-defined set of dependencies as defined + Allows installing a pre-defined set of dependencies as defined in \${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to deploy specific library versions and/or configurations, using a recognizable name. Optionally, PROFILE_NAME may include @@ -83,15 +85,19 @@ optional arguments: that specific version. If unspecified, the 'default:latest' profile will be used, which should include all the possible dependencies. -d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] - build and install a specific dependency, ignoring any --profile + Build and install a specific dependency, ignoring any --profile option provided. If PROFILE_NAME is unspecified, the 'default' profile will be used. Similarly, if PROFILE_VERSION is unspecified, the 'latest' version of the specified profile will be used. -j, --compilecores COMPILE_CORES - number of cores that are used to compile the dependencies - defaults to number of available cores + Set the number of cores that will be used to compile the + dependencies. If unspecified, defaults to the number of + available cores. -t, --test Perform libraries tests. + -P, --profiles-dir PROFILES_DIR + Choose the directory to be used when searching for profiles. + If unspecified, PROFILES_DIR defaults to \${PWD}/profiles. -n, --dry-run Do not actually run, print only what would be done. " @@ -361,6 +367,16 @@ while [[ $# -gt 0 ]]; do PERFORM_TEST=true shift ;; + -P | --profiles-dir) + + if [[ ! -d "$2" ]]; then + echo "ERROR: PROFILES_DIR '$2' does not exist or is not a directory." + exit 1 + fi + + PROFILES_DIR="$2" + shift + ;; -l | --list-dependencies) if [[ -z "$2" ]]; then list_versions diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 5ef23f1a9..83001ef84 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -217,6 +217,9 @@ load_profile() { for k in "${!wgetdeps[@]}"; do PROFILE_WGETDEPS["${k}"]="${wgetdeps[${k}]}" done + + # load source URLs for dependencies + load_sources } load_sources() { @@ -300,7 +303,9 @@ usage_short() { echo " usage: dl_dep.sh -p PROFILE_NAME[:PROFILE_VERSION] | -d DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] | - -l [[PROFILE_NAME:]PROFILE_VERSION] | -n | -h + -l [[PROFILE_NAME:]PROFILE_VERSION] | + -h + [ -P PROFILES_DIR ] [ -n ] DESTINATION_PATH " } @@ -312,15 +317,15 @@ help_msg() { This script gets all GekkoFS dependency sources (excluding the fs itself) positional arguments: - DESTINATION_PATH path where dependencies should be downloaded + DESTINATION_PATH Path where dependencies should be downloaded optional arguments: - -h, --help shows this help message and exits + -h, --help Shows this help message and exits -l, --list-dependencies [[PROFILE_NAME:]PROFILE_VERSION] - list dependency configuration profiles available for download + List dependency configuration profiles available for download -p, --profile PROFILE_NAME[:PROFILE_VERSION] - allows downloading a pre-defined set of dependencies as defined + Allows downloading a pre-defined set of dependencies as defined in \${PROFILES_DIR}/PROFILE_NAME.specs. This is useful to deploy specific library versions and/or configurations, using a recognizable name. Optionally, PROFILE_NAME may include @@ -329,10 +334,13 @@ optional arguments: that specific version. If unspecified, the 'default:latest' profile will be used, which should include all the possible dependencies. -d, --dependency DEPENDENCY_NAME[[@PROFILE_NAME][:PROFILE_VERSION]] - build and install a specific dependency, ignoring any --profile + Build and install a specific dependency, ignoring any --profile option provided. If PROFILE_NAME is unspecified, the 'default' profile will be used. Similarly, if PROFILE_VERSION is unspecified, the 'latest' version of the specified profile will be used. + -P, --profiles-dir PROFILES_DIR + Choose the directory to be used when searching for profiles. + If unspecified, PROFILES_DIR defaults to \${PWD}/profiles. -n, --dry-run Do not actually run, print only what would be done. -v, --verbose Increase download verbosity " @@ -347,9 +355,6 @@ exec_mode_error() { # load default profile for now, might be overridden later load_profile "${DEFAULT_PROFILE}" "${DEFAULT_VERSION}" -# load source URLs for dependencies -load_sources - POSITIONAL=() while [[ $# -gt 0 ]]; do key="$1" @@ -442,6 +447,17 @@ while [[ $# -gt 0 ]]; do VERBOSE=true shift # past argument ;; + -P | --profiles-dir) + + if [[ ! -d "$2" ]]; then + echo "ERROR: PROFILES_DIR '$2' does not exist or is not a directory." + exit 1 + fi + + PROFILES_DIR="$2" + SOURCES_FILE="${PROFILES_DIR}/sources.list" + shift + ;; -n | --dry-run) DRY_RUN=true shift -- GitLab From 82edc70622e610ac9de512abb10fdf6c537069be Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 8 Nov 2021 18:46:46 +0100 Subject: [PATCH 11/19] dep_scripts: Support suffixes to identify multiple dependency flavors --- scripts/dl_dep.sh | 20 +++---- scripts/profiles/0.8.0/default.specs | 12 ++-- .../install/libfabric:experimental.install | 5 +- .../0.8.0/install/libfabric:verbs.install | 5 +- .../install/rocksdb:experimental.install | 56 +++++++++++++++++++ scripts/profiles/sources.list | 1 + tests/scripts/dl_dep.sh/0.8.0/default.out | 6 +- tests/scripts/dl_dep.sh/0.8.0/mogon2.out | 2 +- 8 files changed, 82 insertions(+), 25 deletions(-) create mode 100644 scripts/profiles/0.8.0/install/rocksdb:experimental.install diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 83001ef84..a12ed6a15 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -502,31 +502,24 @@ for dep_name in "${PROFILE_DEP_LIST[@]}"; do if [[ ! -z "${PROFILE_WGETDEPS[${dep_name}]:-}" ]]; then - # dependency names can include a TAG after a colon (e.g. ofi:verbs), - # remove it - dep_id=${dep_name%%:*} - # find required version for dependency dep_version="${PROFILE_WGETDEPS[${dep_name}]}" # build URL for dependency - dep_url="${PROFILE_SOURCES[${dep_id}]}" + dep_url="${PROFILE_SOURCES[${dep_name}]}" if [[ -z "${dep_url}" ]]; then - echo "Missing source URL for '${dep_id}'. Verify ${SOURCES_FILE}." + echo "Missing source URL for '${dep_name}'. Verify ${SOURCES_FILE}." + wait exit 1 fi dep_url="${dep_url/\{\{VERSION\}\}/${dep_version}}" - wgetdeps "${dep_id}" "${dep_url}" & + wgetdeps "${dep_name}" "${dep_url}" & elif [[ ! -z "${PROFILE_CLONEDEPS[${dep_name}]:-}" ]]; then - # dependency names can include a TAG after a colon (e.g. ofi:verbs), - # remove it - dep_id=${dep_name%%:*} - dep_args="" # find required version for dependency @@ -540,10 +533,11 @@ for dep_name in "${PROFILE_DEP_LIST[@]}"; do fi # build URL for dependency - dep_url="${PROFILE_SOURCES[${dep_id}]}" + dep_url="${PROFILE_SOURCES[${dep_name}]}" if [[ -z "${dep_url}" ]]; then - echo "Missing source URL for '${dep_id}'. Verify ${SOURCES_FILE}." + echo "Missing source URL for '${dep_name}'. Verify ${SOURCES_FILE}." + wait exit 1 fi diff --git a/scripts/profiles/0.8.0/default.specs b/scripts/profiles/0.8.0/default.specs index 4512c95b2..a9757f262 100644 --- a/scripts/profiles/0.8.0/default.specs +++ b/scripts/profiles/0.8.0/default.specs @@ -52,8 +52,8 @@ wgetdeps=( clonedeps=( ["bmi"]="6ea0b78fce1b964e45102828cdd05df7040a94c8" ["libfabric"]="HEAD@v1.8.1" -# ["libfabric:experimental"]="HEAD@v1.9.1" -# ["libfabric:verbs"]="HEAD@v1.7.2" + ["libfabric:experimental"]="HEAD@v1.9.1" + ["libfabric:verbs"]="HEAD@v1.7.2" ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" ["margo"]="v0.6.3" ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" @@ -75,8 +75,8 @@ clonedeps_patches=( order=( "bzip2" "zstd" "lz4" "snappy" "capstone" "bmi" "libfabric" - #"libfabric:experimental" - #"libfabric:verbs" + "libfabric:experimental" + "libfabric:verbs" "mercury" "argobots" "margo" "rocksdb" "rocksdb:experimental" "syscall_intercept" "date" "psm2" "agios" ) @@ -91,6 +91,6 @@ order=( # - PERFORM_TEST: whether tests for the package should be executed extra_install_args=( ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" - #["libfabric:verbs"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes" - #["libfabric:experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes" + ["libfabric:verbs"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes" + ["libfabric:experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes" ) diff --git a/scripts/profiles/0.8.0/install/libfabric:experimental.install b/scripts/profiles/0.8.0/install/libfabric:experimental.install index 09bf2a089..9a9da0c93 100644 --- a/scripts/profiles/0.8.0/install/libfabric:experimental.install +++ b/scripts/profiles/0.8.0/install/libfabric:experimental.install @@ -37,11 +37,14 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/libfabric" + CURR="${SOURCE_DIR}/libfabric:experimental" prepare_build_dir "${CURR}" cd "${CURR}" ./autogen.sh diff --git a/scripts/profiles/0.8.0/install/libfabric:verbs.install b/scripts/profiles/0.8.0/install/libfabric:verbs.install index 09bf2a089..59a40b817 100644 --- a/scripts/profiles/0.8.0/install/libfabric:verbs.install +++ b/scripts/profiles/0.8.0/install/libfabric:verbs.install @@ -37,11 +37,14 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/libfabric" + CURR="${SOURCE_DIR}/libfabric:verbs" prepare_build_dir "${CURR}" cd "${CURR}" ./autogen.sh diff --git a/scripts/profiles/0.8.0/install/rocksdb:experimental.install b/scripts/profiles/0.8.0/install/rocksdb:experimental.install new file mode 100644 index 000000000..0f8b54ab2 --- /dev/null +++ b/scripts/profiles/0.8.0/install/rocksdb:experimental.install @@ -0,0 +1,56 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR=${SOURCE_DIR}/rocksdb:experimental + cd "${CURR}" + make clean + PORTABLE=1 USE_SSE=1 USE_RTTI=1 make -j"${CORES}" static_lib + INSTALL_PATH="${INSTALL_DIR}/rocksdb:experimental" make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/sources.list b/scripts/profiles/sources.list index 4485f3d77..354e2e636 100644 --- a/scripts/profiles/sources.list +++ b/scripts/profiles/sources.list @@ -38,6 +38,7 @@ sources=( ["capstone"]="https://github.com/aquynh/capstone/archive/{{VERSION}}.tar.gz" ["argobots"]="https://github.com/pmodels/argobots/archive/v{{VERSION}}.tar.gz" ["rocksdb"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz" + ["rocksdb:experimental"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz" ["psm2"]="https://github.com/intel/opa-psm2/archive/PSM2_{{VERSION}}.tar.gz" ["bmi"]="https://github.com/radix-io/bmi/" ["libfabric"]="https://github.com/ofiwg/libfabric.git" diff --git a/tests/scripts/dl_dep.sh/0.8.0/default.out b/tests/scripts/dl_dep.sh/0.8.0/default.out index 967b9cb50..538da36b2 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/default.out +++ b/tests/scripts/dl_dep.sh/0.8.0/default.out @@ -3,13 +3,13 @@ Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' -Cloned 'https://github.com/radix-io/bmi/' to 'bmi' with commit '[6ea0b78fce1b964e45102828cdd05df7040a94c8]' and flags '' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '-b v1.8.1' -Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:experimental' with commit '[HEAD]' and flags '-b v1.9.1' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:verbs' with commit '[HEAD]' and flags '-b v1.7.2' Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb:experimental' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' diff --git a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out index 154ef6a3b..18815ed93 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out @@ -7,7 +7,7 @@ Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:experimental' with Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb:experimental' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' -- GitLab From 0c2a86cb64df8e4ed9669ca787812a8fca66897a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 8 Nov 2021 18:51:57 +0100 Subject: [PATCH 12/19] dep_scripts: Propagate compiler information --- scripts/compile_dep.sh | 48 +++++++++++++++++++ scripts/profiles/0.8.0/install/agios.install | 3 ++ .../profiles/0.8.0/install/argobots.install | 3 ++ scripts/profiles/0.8.0/install/bmi.install | 3 ++ scripts/profiles/0.8.0/install/bzip2.install | 3 ++ .../profiles/0.8.0/install/capstone.install | 3 ++ scripts/profiles/0.8.0/install/date.install | 3 ++ .../profiles/0.8.0/install/libfabric.install | 3 ++ scripts/profiles/0.8.0/install/lz4.install | 3 ++ scripts/profiles/0.8.0/install/margo.install | 3 ++ .../profiles/0.8.0/install/mercury.install | 3 ++ .../profiles/0.8.0/install/rocksdb.install | 13 ++++- scripts/profiles/0.8.0/install/snappy.install | 3 ++ .../0.8.0/install/syscall_intercept.install | 3 ++ scripts/profiles/0.8.0/install/zstd.install | 3 ++ 15 files changed, 99 insertions(+), 1 deletion(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index a90df71e2..04656efcb 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -284,6 +284,53 @@ find_cmake() { echo "${CMAKE}" } +determine_compiler() { + + compiler_is_gnu() { + COMPILER_NAME="g++" + + if ! COMPILER_FULL_VERSION="$(g++ -dumpfullversion 2>&1)"; then + echo -e "ERROR: Failed to determine compiler version." + echo -e ">> ${COMPILER_FULL_VERSION}" + exit 1 + fi + + COMPILER_MAJOR_VERSION="${COMPILER_FULL_VERSION%%.*}" + } + + compiler_is_clang() { + COMPILER_NAME="clang" + + if ! COMPILER_FULL_VERSION="$(clang -dumpversion 2>&1)"; then + echo -e "ERROR: Failed to determine compiler version." + echo -e ">> ${COMPILER_FULL_VERSION}" + exit 1 + fi + + COMPILER_MAJOR_VERSION="${COMPILER_FULL_VERSION%%.*}" + } + + # We honor the CXX environment variable if defined. + # Otherwise, we try to find the compiler by using `command -v`. + if [[ -n "${CXX}" && ! "${CXX}" =~ ^(g\+\+|clang)$ ]]; then + echo "ERROR: Unknown compiler '${CXX}'" + exit 1 + fi + + if [[ -n "${CXX}" && "${CXX}" =~ ^g\+\+$ ]]; then + compiler_is_gnu + elif [[ -n "${CXX}" && "$CXX" =~ ^clang$ ]]; then + compiler_is_clang + elif [[ $(command -v g++) ]]; then + compiler_is_gnu + elif [[ $(command -v clang) ]]; then + compiler_is_clang + else + echo "ERROR: Unable to determine compiler." + exit 1 + fi +} + POSITIONAL=() while [[ $# -gt 0 ]]; do @@ -471,6 +518,7 @@ for dep_name in "${PROFILE_DEP_LIST[@]}"; do fi if [[ "$DRY_RUN" == false ]]; then + determine_compiler pkg_install [ "${PERFORM_TEST}" ] && pkg_check diff --git a/scripts/profiles/0.8.0/install/agios.install b/scripts/profiles/0.8.0/install/agios.install index 4ceffc2c3..44842bec7 100644 --- a/scripts/profiles/0.8.0/install/agios.install +++ b/scripts/profiles/0.8.0/install/agios.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ diff --git a/scripts/profiles/0.8.0/install/argobots.install b/scripts/profiles/0.8.0/install/argobots.install index 17c4d2be9..865c8b2cb 100644 --- a/scripts/profiles/0.8.0/install/argobots.install +++ b/scripts/profiles/0.8.0/install/argobots.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ diff --git a/scripts/profiles/0.8.0/install/bmi.install b/scripts/profiles/0.8.0/install/bmi.install index 411b4d880..f20650c5d 100644 --- a/scripts/profiles/0.8.0/install/bmi.install +++ b/scripts/profiles/0.8.0/install/bmi.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ diff --git a/scripts/profiles/0.8.0/install/bzip2.install b/scripts/profiles/0.8.0/install/bzip2.install index fa40bc265..c168fc89b 100644 --- a/scripts/profiles/0.8.0/install/bzip2.install +++ b/scripts/profiles/0.8.0/install/bzip2.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ diff --git a/scripts/profiles/0.8.0/install/capstone.install b/scripts/profiles/0.8.0/install/capstone.install index 6a4386761..f91f2560d 100644 --- a/scripts/profiles/0.8.0/install/capstone.install +++ b/scripts/profiles/0.8.0/install/capstone.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ diff --git a/scripts/profiles/0.8.0/install/date.install b/scripts/profiles/0.8.0/install/date.install index cab34851a..336752cc2 100644 --- a/scripts/profiles/0.8.0/install/date.install +++ b/scripts/profiles/0.8.0/install/date.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ diff --git a/scripts/profiles/0.8.0/install/libfabric.install b/scripts/profiles/0.8.0/install/libfabric.install index 09bf2a089..be1214ec9 100644 --- a/scripts/profiles/0.8.0/install/libfabric.install +++ b/scripts/profiles/0.8.0/install/libfabric.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ diff --git a/scripts/profiles/0.8.0/install/lz4.install b/scripts/profiles/0.8.0/install/lz4.install index 26e549ba3..07d4b9162 100644 --- a/scripts/profiles/0.8.0/install/lz4.install +++ b/scripts/profiles/0.8.0/install/lz4.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ diff --git a/scripts/profiles/0.8.0/install/margo.install b/scripts/profiles/0.8.0/install/margo.install index ea1a51d5b..40b40cb50 100644 --- a/scripts/profiles/0.8.0/install/margo.install +++ b/scripts/profiles/0.8.0/install/margo.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ diff --git a/scripts/profiles/0.8.0/install/mercury.install b/scripts/profiles/0.8.0/install/mercury.install index d2677fd63..b1546b4b1 100644 --- a/scripts/profiles/0.8.0/install/mercury.install +++ b/scripts/profiles/0.8.0/install/mercury.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ diff --git a/scripts/profiles/0.8.0/install/rocksdb.install b/scripts/profiles/0.8.0/install/rocksdb.install index 7e14bbaab..a7abf8c9d 100644 --- a/scripts/profiles/0.8.0/install/rocksdb.install +++ b/scripts/profiles/0.8.0/install/rocksdb.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ @@ -44,7 +47,15 @@ pkg_install() { CURR=${SOURCE_DIR}/rocksdb cd "${CURR}" make clean - PORTABLE=1 USE_SSE=1 USE_RTTI=1 make -j"${CORES}" static_lib + + # gcc 9 and clang 8 need -Wno-error=deprecated-copy -Wno-error=pessimizing-move + if [[ ("${COMPILER_NAME}" == "g++" && "${COMPILER_MAJOR_VERSION}" -ge 9) || + ("${COMPILER_NAME}" == "clang" && "${COMPILER_MAJOR_VERSION}" -ge 8) ]]; then + CXXFLAGS='-Wno-error=deprecated-copy -Wno-error=pessimizing-move' PORTABLE=1 USE_SSE=1 USE_RTTI=1 make -j"${CORES}" static_lib + else + PORTABLE=1 USE_SSE=1 USE_RTTI=1 make -j"${CORES}" static_lib + fi + INSTALL_PATH="${INSTALL_DIR}" make install } diff --git a/scripts/profiles/0.8.0/install/snappy.install b/scripts/profiles/0.8.0/install/snappy.install index 078c72851..a0dcf8581 100644 --- a/scripts/profiles/0.8.0/install/snappy.install +++ b/scripts/profiles/0.8.0/install/snappy.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ diff --git a/scripts/profiles/0.8.0/install/syscall_intercept.install b/scripts/profiles/0.8.0/install/syscall_intercept.install index 72d5551a6..72680b988 100644 --- a/scripts/profiles/0.8.0/install/syscall_intercept.install +++ b/scripts/profiles/0.8.0/install/syscall_intercept.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ diff --git a/scripts/profiles/0.8.0/install/zstd.install b/scripts/profiles/0.8.0/install/zstd.install index 2ab403dbf..2eea78feb 100644 --- a/scripts/profiles/0.8.0/install/zstd.install +++ b/scripts/profiles/0.8.0/install/zstd.install @@ -37,6 +37,9 @@ ## downloaded ## - INSTALL_DIR: the directory where the package should be installed ## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) ## - PERFORM_TEST: whether tests for the package should be executed ################################################################################ -- GitLab From bec05bbb5f216ad9bec7bd031090859592aac845 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 22 Nov 2021 13:00:17 +0100 Subject: [PATCH 13/19] dep_scripts: Fix error when no profile or version selected --- scripts/dl_dep.sh | 5 ++--- tests/scripts/compile_dep.sh/0.8.0/default.out | 1 - tests/scripts/compile_dep.sh/0.8.0/mogon2.out | 1 - tests/scripts/compile_dep.sh/test.bats | 14 ++++++++++++++ tests/scripts/dl_dep.sh/test.bats | 14 ++++++++++++++ 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index a12ed6a15..f09d70c19 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -39,6 +39,8 @@ DRY_RUN=false DEFAULT_PROFILE="default" DEFAULT_VERSION="latest" +PROFILE_NAME=${DEFAULT_PROFILE} +PROFILE_VERSION=${DEFAULT_VERSION} PROFILES_DIR="${PWD}/profiles" SOURCES_FILE="${PROFILES_DIR}/sources.list" declare -a PROFILE_DEP_LIST @@ -390,9 +392,6 @@ while [[ $# -gt 0 ]]; do exit 1 fi - PROFILE_NAME=${DEFAULT_PROFILE} - PROFILE_VERSION=${DEFAULT_VERSION} - # e.g. mercury@mogon1:latest if [[ "$2" =~ ^(.*)@(.*):(.*)$ ]]; then if [[ -n "${BASH_REMATCH[1]}" ]]; then diff --git a/tests/scripts/compile_dep.sh/0.8.0/default.out b/tests/scripts/compile_dep.sh/0.8.0/default.out index dfd76cf06..454ac0da8 100644 --- a/tests/scripts/compile_dep.sh/0.8.0/default.out +++ b/tests/scripts/compile_dep.sh/0.8.0/default.out @@ -10,7 +10,6 @@ ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: rocksdb:experimental ############################### -WARNING: Install script for 'rocksdb:experimental' not found. Skipping. ######## Installing: syscall_intercept ############################### ######## Installing: date ############################### ######## Installing: psm2 ############################### diff --git a/tests/scripts/compile_dep.sh/0.8.0/mogon2.out b/tests/scripts/compile_dep.sh/0.8.0/mogon2.out index b18dc3762..4004cb9fa 100644 --- a/tests/scripts/compile_dep.sh/0.8.0/mogon2.out +++ b/tests/scripts/compile_dep.sh/0.8.0/mogon2.out @@ -8,7 +8,6 @@ ######## Installing: argobots ############################### ######## Installing: margo ############################### ######## Installing: rocksdb:experimental ############################### -WARNING: Install script for 'rocksdb:experimental' not found. Skipping. ######## Installing: syscall_intercept ############################### ######## Installing: date ############################### ######## Installing: psm2 ############################### diff --git a/tests/scripts/compile_dep.sh/test.bats b/tests/scripts/compile_dep.sh/test.bats index ca994bc44..244aff136 100644 --- a/tests/scripts/compile_dep.sh/test.bats +++ b/tests/scripts/compile_dep.sh/test.bats @@ -29,6 +29,20 @@ teardown() { assert_output --partial "usage: compile_dep.sh" } +@test "[compile_dep.sh] Check no --profile or --dependency options" { + + expected_output=${SCRIPT_TESTDIR}/latest/default.out + + assert_exist $expected_output + + run compile_dep.sh -n bar baz + + while IFS= read -r line + do + assert_output --partial "$line" + done < "$expected_output" +} + @test "[compile_dep.sh] Check --profile and --dependency option at the same time" { run -1 compile_dep.sh -p foobar -d barbaz assert_output --partial "ERROR: --profile and --dependency options are mutually exclusive" diff --git a/tests/scripts/dl_dep.sh/test.bats b/tests/scripts/dl_dep.sh/test.bats index 784ac2b8b..51b27ba3a 100644 --- a/tests/scripts/dl_dep.sh/test.bats +++ b/tests/scripts/dl_dep.sh/test.bats @@ -29,6 +29,20 @@ teardown() { assert_output --partial "usage: dl_dep.sh" } +@test "[dl_dep.sh] Check no --profile or --dependency options" { + + expected_output=${SCRIPT_TESTDIR}/latest/default.out + + assert_exist $expected_output + + run dl_dep.sh -n "${BATS_TEST_TMPDIR}" + + while IFS= read -r line + do + assert_output --partial "$line" + done < "$expected_output" +} + @test "[dl_dep.sh] Check --profile and --dependency option at the same time" { run -1 dl_dep.sh -p foobar -d barbaz assert_output --partial "ERROR: --profile and --dependency options are mutually exclusive" -- GitLab From 1b2b225bee01f89bd04033eb3f8c603837f2d85c Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 22 Nov 2021 13:15:31 +0100 Subject: [PATCH 14/19] dep_scripts: Fix wrong directory when applying patches (#184) --- scripts/dl_dep.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index f09d70c19..53665315f 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -268,7 +268,7 @@ clonedeps() { # apply patch if provided if [[ -n "${PATCH}" ]]; then - [[ "$DRY_RUN" == true ]] || git apply --verbose "${PATCH_DIR}/${PATCH}" + [[ "$DRY_RUN" == true ]] || (cd "${SOURCE_DIR}/${FOLDER}" && git apply --verbose "${PATCH_DIR}/${PATCH}" ) fi } -- GitLab From 9a5acdde69d779210ec607a0fbd4578d68f02c1e Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 22 Nov 2021 18:40:57 +0100 Subject: [PATCH 15/19] dep_scripts: Fix bug passing multiple args to git clone --- scripts/dl_dep.sh | 28 ++++++++++++++--------- tests/scripts/dl_dep.sh/0.8.0/all.out | 4 ++-- tests/scripts/dl_dep.sh/0.8.0/ci.out | 4 ++-- tests/scripts/dl_dep.sh/0.8.0/default.out | 8 +++---- tests/scripts/dl_dep.sh/0.8.0/mogon1.out | 2 +- tests/scripts/dl_dep.sh/0.8.0/mogon2.out | 2 +- tests/scripts/dl_dep.sh/0.8.0/ngio.out | 4 ++-- 7 files changed, 29 insertions(+), 23 deletions(-) diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 53665315f..317b6fff5 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -247,11 +247,12 @@ clonedeps() { fi trap exit_child EXIT - local FOLDER=$1 - local REPO=$2 - local COMMIT=$3 - local GIT_FLAGS=$4 - local PATCH=$5 + local FOLDER="$1" + local REPO="$2" + local COMMIT="$3" + local PATCH="$4" + shift 4 + local GIT_FLAGS=("$@") local ACTION @@ -259,12 +260,12 @@ clonedeps() { [[ "$DRY_RUN" == true ]] || (cd "${SOURCE_DIR}/${FOLDER}" && git fetch -q) ACTION="Pulled" else - [[ "$DRY_RUN" == true ]] || (git clone ${COMMON_GIT_FLAGS} ${GIT_FLAGS} -- "${REPO}" "${SOURCE_DIR}/${FOLDER}") + [[ "$DRY_RUN" == true ]] || (git clone ${COMMON_GIT_FLAGS} "${GIT_FLAGS[@]}" -- "${REPO}" "${SOURCE_DIR}/${FOLDER}") ACTION="Cloned" fi # fix the version [[ "$DRY_RUN" == true ]] || (cd "${SOURCE_DIR}/${FOLDER}" && git checkout -qf "${COMMIT}") - echo "${ACTION} '${REPO}' to '${FOLDER}' with commit '[${COMMIT}]' and flags '${GIT_FLAGS}'" + echo "${ACTION} '${REPO}' to '${FOLDER}' with commit '[${COMMIT}]' and flags '${GIT_FLAGS[@]}'" # apply patch if provided if [[ -n "${PATCH}" ]]; then @@ -519,7 +520,7 @@ for dep_name in "${PROFILE_DEP_LIST[@]}"; do elif [[ ! -z "${PROFILE_CLONEDEPS[${dep_name}]:-}" ]]; then - dep_args="" + dep_args=() # find required version for dependency dep_version="${PROFILE_CLONEDEPS[${dep_name}]}" @@ -527,7 +528,7 @@ for dep_name in "${PROFILE_DEP_LIST[@]}"; do # version may be a commit hash, a tag or something like HEAD@BRANCH_NAME # if it's the latter, remove the @BRANCH_NAME if [[ "${dep_version}" =~ ^(.*)@(.*)$ ]]; then - dep_args+="-b ${BASH_REMATCH[2]}" + dep_args+=("--branch=${BASH_REMATCH[2]}") dep_version=${BASH_REMATCH[1]} fi @@ -543,11 +544,16 @@ for dep_name in "${PROFILE_DEP_LIST[@]}"; do dep_url="${dep_url/\{\{VERSION\}\}/${dep_version}}" # check if extra args are required - dep_args+="${PROFILE_CLONEDEPS_ARGS[${dep_name}]}" + for arg in "${PROFILE_CLONEDEPS_ARGS[${dep_name}]}"; + do + if [[ -n "${arg}" ]]; then + dep_args+=("${arg}") + fi + done dep_patch=${PROFILE_CLONEDEPS_PATCHES[${dep_name}]} - clonedeps "${dep_name}" "${dep_url}" "${dep_version}" "${dep_args}" "${dep_patch}" & + clonedeps "${dep_name}" "${dep_url}" "${dep_version}" "${dep_patch}" "${dep_args[@]}" & else echo "Unknown dependency '${dep_name}'." diff --git a/tests/scripts/dl_dep.sh/0.8.0/all.out b/tests/scripts/dl_dep.sh/0.8.0/all.out index 9253d8407..3c3328b01 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/all.out +++ b/tests/scripts/dl_dep.sh/0.8.0/all.out @@ -3,12 +3,12 @@ Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' Cloned 'https://github.com/radix-io/bmi/' to 'bmi' with commit '[6ea0b78fce1b964e45102828cdd05df7040a94c8]' and flags '' Downloaded 'https://sourceforge.net/projects/bzip2/files/bzip2-1.0.6.tar.gz' to 'bzip2' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '-b v1.8.1' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.8.1' Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '-b development' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.8.0/ci.out b/tests/scripts/dl_dep.sh/0.8.0/ci.out index 93c1eee29..60e258c98 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/ci.out +++ b/tests/scripts/dl_dep.sh/0.8.0/ci.out @@ -1,8 +1,8 @@ -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '-b v1.8.1' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.8.1' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '-b development' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' diff --git a/tests/scripts/dl_dep.sh/0.8.0/default.out b/tests/scripts/dl_dep.sh/0.8.0/default.out index 538da36b2..2189a3b3b 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/default.out +++ b/tests/scripts/dl_dep.sh/0.8.0/default.out @@ -3,9 +3,9 @@ Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '-b v1.8.1' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:experimental' with commit '[HEAD]' and flags '-b v1.9.1' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:verbs' with commit '[HEAD]' and flags '-b v1.7.2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.8.1' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:verbs' with commit '[HEAD]' and flags '--branch=v1.7.2' Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' @@ -13,4 +13,4 @@ Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rock Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' -Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '-b development' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' diff --git a/tests/scripts/dl_dep.sh/0.8.0/mogon1.out b/tests/scripts/dl_dep.sh/0.8.0/mogon1.out index 9a88c8313..c57e7b49a 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/mogon1.out +++ b/tests/scripts/dl_dep.sh/0.8.0/mogon1.out @@ -2,7 +2,7 @@ Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:verbs' with commit '[HEAD]' and flags '-b v1.7.2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:verbs' with commit '[HEAD]' and flags '--branch=v1.7.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' diff --git a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out index 18815ed93..2d38d6bbe 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out @@ -3,7 +3,7 @@ Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:experimental' with commit '[HEAD]' and flags '-b v1.9.1' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' diff --git a/tests/scripts/dl_dep.sh/0.8.0/ngio.out b/tests/scripts/dl_dep.sh/0.8.0/ngio.out index 6fd8740fc..8bd3573aa 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/ngio.out +++ b/tests/scripts/dl_dep.sh/0.8.0/ngio.out @@ -2,7 +2,7 @@ Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:experimental' with commit '[HEAD]' and flags '-b v1.9.1' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' @@ -10,4 +10,4 @@ Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocks Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '-b development' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' -- GitLab From be58ff991aeb6f79de35ee10af58e734a7f418e6 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 25 Nov 2021 11:47:33 +0100 Subject: [PATCH 16/19] dep_scripts: Better handling of scripts PWD --- scripts/compile_dep.sh | 4 ++-- scripts/dl_dep.sh | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index 04656efcb..ce17de93a 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -27,7 +27,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ -PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PATCH_DIR="${PATCH_DIR}/patches" DEPENDENCY="" CORES="" @@ -42,7 +42,7 @@ DEFAULT_PROFILE="default" DEFAULT_VERSION="latest" PROFILE_NAME=${DEFAULT_PROFILE} PROFILE_VERSION=${DEFAULT_VERSION} -PROFILES_DIR="${PWD}/profiles" +PROFILES_DIR="${SCRIPT_DIR}/profiles" declare -a PROFILE_DEP_LIST declare -A PROFILE_DEP_NAMES declare -A PROFILE_WGETDEPS PROFILE_CLONEDEPS PROFILE_SOURCES PROFILE_EXTRA_INSTALL_ARGS diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 317b6fff5..26e9caba9 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -29,8 +29,8 @@ COMMON_CURL_FLAGS="--silent --fail --show-error --location -O" COMMON_GIT_FLAGS="--quiet --single-branch -c advice.detachedHead=false" -PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PATCH_DIR="${PATCH_DIR}/patches" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PATCH_DIR="${SCRIPT_DIR}/patches" DEPENDENCY="" EXECUTION_MODE= @@ -41,7 +41,7 @@ DEFAULT_PROFILE="default" DEFAULT_VERSION="latest" PROFILE_NAME=${DEFAULT_PROFILE} PROFILE_VERSION=${DEFAULT_VERSION} -PROFILES_DIR="${PWD}/profiles" +PROFILES_DIR="${SCRIPT_DIR}/profiles" SOURCES_FILE="${PROFILES_DIR}/sources.list" declare -a PROFILE_DEP_LIST declare -A PROFILE_DEP_NAMES -- GitLab From 964c930baa5da5b41f5de2fbf9411f7c73e57e27 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 25 Nov 2021 12:03:37 +0100 Subject: [PATCH 17/19] dep_scripts: Change suffix separator to '%' in dep flavors --- scripts/profiles/0.8.0/default.specs | 16 ++++++++-------- scripts/profiles/0.8.0/install/agios.install | 3 ++- scripts/profiles/0.8.0/install/argobots.install | 3 ++- scripts/profiles/0.8.0/install/bmi.install | 3 ++- scripts/profiles/0.8.0/install/bzip2.install | 3 ++- scripts/profiles/0.8.0/install/capstone.install | 3 ++- scripts/profiles/0.8.0/install/date.install | 3 ++- ...bs.install => libfabric%experimental.install} | 5 +++-- ...erimental.install => libfabric%verbs.install} | 5 +++-- scripts/profiles/0.8.0/install/libfabric.install | 5 +++-- scripts/profiles/0.8.0/install/lz4.install | 3 ++- scripts/profiles/0.8.0/install/margo.install | 3 ++- scripts/profiles/0.8.0/install/mercury.install | 3 ++- ...ntal.install => rocksdb%experimental.install} | 5 +++-- scripts/profiles/0.8.0/install/rocksdb.install | 3 ++- scripts/profiles/0.8.0/install/snappy.install | 3 ++- .../0.8.0/install/syscall_intercept.install | 3 ++- scripts/profiles/0.8.0/install/zstd.install | 3 ++- scripts/profiles/0.8.0/mogon1.specs | 6 +++--- scripts/profiles/0.8.0/mogon2.specs | 10 +++++----- scripts/profiles/0.8.0/ngio.specs | 6 +++--- scripts/profiles/sources.list | 6 +++--- tests/scripts/compile_dep.sh/0.8.0/default.out | 2 +- tests/scripts/compile_dep.sh/0.8.0/mogon1.out | 2 +- tests/scripts/compile_dep.sh/0.8.0/mogon2.out | 4 ++-- tests/scripts/compile_dep.sh/0.8.0/ngio.out | 2 +- tests/scripts/dl_dep.sh/0.8.0/default.out | 6 +++--- tests/scripts/dl_dep.sh/0.8.0/mogon1.out | 2 +- tests/scripts/dl_dep.sh/0.8.0/mogon2.out | 4 ++-- tests/scripts/dl_dep.sh/0.8.0/ngio.out | 2 +- 30 files changed, 72 insertions(+), 55 deletions(-) rename scripts/profiles/0.8.0/install/{libfabric:verbs.install => libfabric%experimental.install} (96%) rename scripts/profiles/0.8.0/install/{libfabric:experimental.install => libfabric%verbs.install} (96%) rename scripts/profiles/0.8.0/install/{rocksdb:experimental.install => rocksdb%experimental.install} (96%) diff --git a/scripts/profiles/0.8.0/default.specs b/scripts/profiles/0.8.0/default.specs index a9757f262..d3506b0d2 100644 --- a/scripts/profiles/0.8.0/default.specs +++ b/scripts/profiles/0.8.0/default.specs @@ -44,7 +44,7 @@ wgetdeps=( ["capstone"]="4.0.1" ["argobots"]="1.0rc1" ["rocksdb"]="6.2.2" - ["rocksdb:experimental"]="6.11.4" + ["rocksdb%experimental"]="6.11.4" ["psm2"]="11.2.86" ) @@ -52,8 +52,8 @@ wgetdeps=( clonedeps=( ["bmi"]="6ea0b78fce1b964e45102828cdd05df7040a94c8" ["libfabric"]="HEAD@v1.8.1" - ["libfabric:experimental"]="HEAD@v1.9.1" - ["libfabric:verbs"]="HEAD@v1.7.2" + ["libfabric%experimental"]="HEAD@v1.9.1" + ["libfabric%verbs"]="HEAD@v1.7.2" ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" ["margo"]="v0.6.3" ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" @@ -75,9 +75,9 @@ clonedeps_patches=( order=( "bzip2" "zstd" "lz4" "snappy" "capstone" "bmi" "libfabric" - "libfabric:experimental" - "libfabric:verbs" - "mercury" "argobots" "margo" "rocksdb" "rocksdb:experimental" + "libfabric%experimental" + "libfabric%verbs" + "mercury" "argobots" "margo" "rocksdb" "rocksdb%experimental" "syscall_intercept" "date" "psm2" "agios" ) @@ -91,6 +91,6 @@ order=( # - PERFORM_TEST: whether tests for the package should be executed extra_install_args=( ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" - ["libfabric:verbs"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes" - ["libfabric:experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes" + ["libfabric%verbs"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes" + ["libfabric%experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes" ) diff --git a/scripts/profiles/0.8.0/install/agios.install b/scripts/profiles/0.8.0/install/agios.install index 44842bec7..f19d03f09 100644 --- a/scripts/profiles/0.8.0/install/agios.install +++ b/scripts/profiles/0.8.0/install/agios.install @@ -44,7 +44,8 @@ ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/agios" + ID="agios" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}/build" ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" .. diff --git a/scripts/profiles/0.8.0/install/argobots.install b/scripts/profiles/0.8.0/install/argobots.install index 865c8b2cb..6c0aff146 100644 --- a/scripts/profiles/0.8.0/install/argobots.install +++ b/scripts/profiles/0.8.0/install/argobots.install @@ -44,7 +44,8 @@ ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/argobots" + ID="argobots" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}" ./autogen.sh diff --git a/scripts/profiles/0.8.0/install/bmi.install b/scripts/profiles/0.8.0/install/bmi.install index f20650c5d..a0cd0b675 100644 --- a/scripts/profiles/0.8.0/install/bmi.install +++ b/scripts/profiles/0.8.0/install/bmi.install @@ -44,7 +44,8 @@ ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/bmi" + ID="bmi" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}" ./prepare diff --git a/scripts/profiles/0.8.0/install/bzip2.install b/scripts/profiles/0.8.0/install/bzip2.install index c168fc89b..a1f45ab39 100644 --- a/scripts/profiles/0.8.0/install/bzip2.install +++ b/scripts/profiles/0.8.0/install/bzip2.install @@ -44,7 +44,8 @@ ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/bzip2" + ID="bzip2" + CURR="${SOURCE_DIR}/${ID}" cd "${CURR}" make install PREFIX="${INSTALL_DIR}" } diff --git a/scripts/profiles/0.8.0/install/capstone.install b/scripts/profiles/0.8.0/install/capstone.install index f91f2560d..60bbd5faf 100644 --- a/scripts/profiles/0.8.0/install/capstone.install +++ b/scripts/profiles/0.8.0/install/capstone.install @@ -44,7 +44,8 @@ ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/capstone" + ID="capstone" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}/build" ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. diff --git a/scripts/profiles/0.8.0/install/date.install b/scripts/profiles/0.8.0/install/date.install index 336752cc2..4adf6bc11 100644 --- a/scripts/profiles/0.8.0/install/date.install +++ b/scripts/profiles/0.8.0/install/date.install @@ -44,7 +44,8 @@ ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/date" + ID="date" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}/build" ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON .. diff --git a/scripts/profiles/0.8.0/install/libfabric:verbs.install b/scripts/profiles/0.8.0/install/libfabric%experimental.install similarity index 96% rename from scripts/profiles/0.8.0/install/libfabric:verbs.install rename to scripts/profiles/0.8.0/install/libfabric%experimental.install index 59a40b817..98cbe5b0c 100644 --- a/scripts/profiles/0.8.0/install/libfabric:verbs.install +++ b/scripts/profiles/0.8.0/install/libfabric%experimental.install @@ -44,14 +44,15 @@ ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/libfabric:verbs" + ID="libfabric%experimental" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}" ./autogen.sh cd "${CURR}/build" OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" - EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS["libfabric"]}" + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" diff --git a/scripts/profiles/0.8.0/install/libfabric:experimental.install b/scripts/profiles/0.8.0/install/libfabric%verbs.install similarity index 96% rename from scripts/profiles/0.8.0/install/libfabric:experimental.install rename to scripts/profiles/0.8.0/install/libfabric%verbs.install index 9a9da0c93..46fe27433 100644 --- a/scripts/profiles/0.8.0/install/libfabric:experimental.install +++ b/scripts/profiles/0.8.0/install/libfabric%verbs.install @@ -44,14 +44,15 @@ ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/libfabric:experimental" + ID="libfabric%verbs" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}" ./autogen.sh cd "${CURR}/build" OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" - EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS["libfabric"]}" + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" diff --git a/scripts/profiles/0.8.0/install/libfabric.install b/scripts/profiles/0.8.0/install/libfabric.install index be1214ec9..deb0f5111 100644 --- a/scripts/profiles/0.8.0/install/libfabric.install +++ b/scripts/profiles/0.8.0/install/libfabric.install @@ -44,14 +44,15 @@ ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/libfabric" + ID=libfabric + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}" ./autogen.sh cd "${CURR}/build" OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" - EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS["libfabric"]}" + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" diff --git a/scripts/profiles/0.8.0/install/lz4.install b/scripts/profiles/0.8.0/install/lz4.install index 07d4b9162..2e807ab88 100644 --- a/scripts/profiles/0.8.0/install/lz4.install +++ b/scripts/profiles/0.8.0/install/lz4.install @@ -44,7 +44,8 @@ ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/lz4" + ID="lz4" + CURR="${SOURCE_DIR}/${ID}" cd "${CURR}" make -j"${CORES}" make DESTDIR="${INSTALL_DIR}" PREFIX="" install diff --git a/scripts/profiles/0.8.0/install/margo.install b/scripts/profiles/0.8.0/install/margo.install index 40b40cb50..8ddfadd98 100644 --- a/scripts/profiles/0.8.0/install/margo.install +++ b/scripts/profiles/0.8.0/install/margo.install @@ -44,7 +44,8 @@ ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/margo" + ID="margo" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}" ./prepare.sh diff --git a/scripts/profiles/0.8.0/install/mercury.install b/scripts/profiles/0.8.0/install/mercury.install index b1546b4b1..16376a66a 100644 --- a/scripts/profiles/0.8.0/install/mercury.install +++ b/scripts/profiles/0.8.0/install/mercury.install @@ -61,7 +61,8 @@ pkg_install() { USE_OFI="-DNA_USE_OFI:BOOL=OFF" fi - CURR="${SOURCE_DIR}/mercury" + ID="mercury" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}/build" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ diff --git a/scripts/profiles/0.8.0/install/rocksdb:experimental.install b/scripts/profiles/0.8.0/install/rocksdb%experimental.install similarity index 96% rename from scripts/profiles/0.8.0/install/rocksdb:experimental.install rename to scripts/profiles/0.8.0/install/rocksdb%experimental.install index 0f8b54ab2..3acd77819 100644 --- a/scripts/profiles/0.8.0/install/rocksdb:experimental.install +++ b/scripts/profiles/0.8.0/install/rocksdb%experimental.install @@ -44,11 +44,12 @@ ################################################################################ pkg_install() { - CURR=${SOURCE_DIR}/rocksdb:experimental + ID="rocksdb%experimental" + CURR="${SOURCE_DIR}/${ID}" cd "${CURR}" make clean PORTABLE=1 USE_SSE=1 USE_RTTI=1 make -j"${CORES}" static_lib - INSTALL_PATH="${INSTALL_DIR}/rocksdb:experimental" make install + INSTALL_PATH="${INSTALL_DIR}/${ID}" make install } pkg_check() { diff --git a/scripts/profiles/0.8.0/install/rocksdb.install b/scripts/profiles/0.8.0/install/rocksdb.install index a7abf8c9d..9959bf1f1 100644 --- a/scripts/profiles/0.8.0/install/rocksdb.install +++ b/scripts/profiles/0.8.0/install/rocksdb.install @@ -44,7 +44,8 @@ ################################################################################ pkg_install() { - CURR=${SOURCE_DIR}/rocksdb + ID="rocksdb" + CURR="${SOURCE_DIR}/${ID}" cd "${CURR}" make clean diff --git a/scripts/profiles/0.8.0/install/snappy.install b/scripts/profiles/0.8.0/install/snappy.install index a0dcf8581..5d9b1ce92 100644 --- a/scripts/profiles/0.8.0/install/snappy.install +++ b/scripts/profiles/0.8.0/install/snappy.install @@ -44,7 +44,8 @@ ################################################################################ pkg_install() { - CURR=${SOURCE_DIR}/snappy + ID="snappy" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}"/build $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. diff --git a/scripts/profiles/0.8.0/install/syscall_intercept.install b/scripts/profiles/0.8.0/install/syscall_intercept.install index 72680b988..aa469255a 100644 --- a/scripts/profiles/0.8.0/install/syscall_intercept.install +++ b/scripts/profiles/0.8.0/install/syscall_intercept.install @@ -44,7 +44,8 @@ ################################################################################ pkg_install() { - CURR=${SOURCE_DIR}/syscall_intercept + ID="syscall_intercept" + CURR="${SOURCE_DIR}/${ID}" prepare_build_dir "${CURR}" cd "${CURR}"/build $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF .. diff --git a/scripts/profiles/0.8.0/install/zstd.install b/scripts/profiles/0.8.0/install/zstd.install index 2eea78feb..def1d467d 100644 --- a/scripts/profiles/0.8.0/install/zstd.install +++ b/scripts/profiles/0.8.0/install/zstd.install @@ -44,7 +44,8 @@ ################################################################################ pkg_install() { - CURR="${SOURCE_DIR}/zstd/build/cmake" + ID="zstd" + CURR="${SOURCE_DIR}/${ID}/build/cmake" prepare_build_dir "${CURR}" cd "${CURR}"/build $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. diff --git a/scripts/profiles/0.8.0/mogon1.specs b/scripts/profiles/0.8.0/mogon1.specs index 65bee82fb..86ee11dc3 100644 --- a/scripts/profiles/0.8.0/mogon1.specs +++ b/scripts/profiles/0.8.0/mogon1.specs @@ -46,7 +46,7 @@ wgetdeps=( # Dependencies that must be cloned clonedeps=( - ["libfabric:verbs"]="HEAD@v1.7.2" + ["libfabric%verbs"]="HEAD@v1.7.2" ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" ["margo"]="v0.6.3" ["rocksdb"]="6.2.2" @@ -66,7 +66,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading/installing order=( - "zstd" "lz4" "snappy" "capstone" "libfabric:verbs" "mercury" + "zstd" "lz4" "snappy" "capstone" "libfabric%verbs" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" ) @@ -79,5 +79,5 @@ order=( # - CORES: the number of cores to use when building # - PERFORM_TEST: whether tests for the package should be executed extra_install_args=( - ["libfabric:verbs"]="--enable-verbs=yes" + ["libfabric%verbs"]="--enable-verbs=yes" ) diff --git a/scripts/profiles/0.8.0/mogon2.specs b/scripts/profiles/0.8.0/mogon2.specs index 0e4306584..ad8d8d97d 100644 --- a/scripts/profiles/0.8.0/mogon2.specs +++ b/scripts/profiles/0.8.0/mogon2.specs @@ -43,13 +43,13 @@ wgetdeps=( ["snappy"]="1.1.7" ["capstone"]="4.0.1" ["argobots"]="1.0rc1" - ["rocksdb:experimental"]="6.11.4" + ["rocksdb%experimental"]="6.11.4" ["psm2"]="11.2.86" ) # Dependencies that must be cloned clonedeps=( - ["libfabric:experimental"]="HEAD@v1.9.1" + ["libfabric%experimental"]="HEAD@v1.9.1" ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" ["margo"]="v0.6.3" ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" @@ -68,8 +68,8 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "bzip2" "zstd" "lz4" "snappy" "capstone" "libfabric:experimental" - "mercury" "argobots" "margo" "rocksdb:experimental" "syscall_intercept" + "bzip2" "zstd" "lz4" "snappy" "capstone" "libfabric%experimental" + "mercury" "argobots" "margo" "rocksdb%experimental" "syscall_intercept" "date" "psm2" ) @@ -82,5 +82,5 @@ order=( # - CORES: the number of cores to use when building # - PERFORM_TEST: whether tests for the package should be executed extra_install_args=( - ["libfabric:experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" + ["libfabric%experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" ) diff --git a/scripts/profiles/0.8.0/ngio.specs b/scripts/profiles/0.8.0/ngio.specs index 715c08a34..1742e8c54 100644 --- a/scripts/profiles/0.8.0/ngio.specs +++ b/scripts/profiles/0.8.0/ngio.specs @@ -48,7 +48,7 @@ wgetdeps=( # Dependencies that must be cloned clonedeps=( - ["libfabric:experimental"]="HEAD@v1.9.1" + ["libfabric%experimental"]="HEAD@v1.9.1" ["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd" ["margo"]="v0.6.3" ["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd" @@ -68,7 +68,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "zstd" "lz4" "snappy" "capstone" "libfabric:experimental" "mercury" + "zstd" "lz4" "snappy" "capstone" "libfabric%experimental" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "psm2" "agios" ) @@ -81,5 +81,5 @@ order=( # - CORES: the number of cores to use when building # - PERFORM_TEST: whether tests for the package should be executed extra_install_args=( - ["libfabric:experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" + ["libfabric%experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" ) diff --git a/scripts/profiles/sources.list b/scripts/profiles/sources.list index 354e2e636..5cbc13017 100644 --- a/scripts/profiles/sources.list +++ b/scripts/profiles/sources.list @@ -38,12 +38,12 @@ sources=( ["capstone"]="https://github.com/aquynh/capstone/archive/{{VERSION}}.tar.gz" ["argobots"]="https://github.com/pmodels/argobots/archive/v{{VERSION}}.tar.gz" ["rocksdb"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz" - ["rocksdb:experimental"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz" + ["rocksdb%experimental"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz" ["psm2"]="https://github.com/intel/opa-psm2/archive/PSM2_{{VERSION}}.tar.gz" ["bmi"]="https://github.com/radix-io/bmi/" ["libfabric"]="https://github.com/ofiwg/libfabric.git" - ["libfabric:experimental"]="https://github.com/ofiwg/libfabric.git" - ["libfabric:verbs"]="https://github.com/ofiwg/libfabric.git" + ["libfabric%experimental"]="https://github.com/ofiwg/libfabric.git" + ["libfabric%verbs"]="https://github.com/ofiwg/libfabric.git" ["mercury"]="https://github.com/mercury-hpc/mercury" ["margo"]="https://xgitlab.cels.anl.gov/sds/margo.git" ["syscall_intercept"]="https://github.com/pmem/syscall_intercept.git" diff --git a/tests/scripts/compile_dep.sh/0.8.0/default.out b/tests/scripts/compile_dep.sh/0.8.0/default.out index 454ac0da8..a1781600a 100644 --- a/tests/scripts/compile_dep.sh/0.8.0/default.out +++ b/tests/scripts/compile_dep.sh/0.8.0/default.out @@ -9,7 +9,7 @@ ######## Installing: argobots ############################### ######## Installing: margo ############################### ######## Installing: rocksdb ############################### -######## Installing: rocksdb:experimental ############################### +######## Installing: rocksdb%experimental ############################### ######## Installing: syscall_intercept ############################### ######## Installing: date ############################### ######## Installing: psm2 ############################### diff --git a/tests/scripts/compile_dep.sh/0.8.0/mogon1.out b/tests/scripts/compile_dep.sh/0.8.0/mogon1.out index 572319b9a..3afaabd70 100644 --- a/tests/scripts/compile_dep.sh/0.8.0/mogon1.out +++ b/tests/scripts/compile_dep.sh/0.8.0/mogon1.out @@ -2,7 +2,7 @@ ######## Installing: lz4 ############################### ######## Installing: snappy ############################### ######## Installing: capstone ############################### -######## Installing: libfabric:verbs ############################### +######## Installing: libfabric%verbs ############################### ######## Installing: mercury ############################### ######## Installing: argobots ############################### ######## Installing: margo ############################### diff --git a/tests/scripts/compile_dep.sh/0.8.0/mogon2.out b/tests/scripts/compile_dep.sh/0.8.0/mogon2.out index 4004cb9fa..925ef005d 100644 --- a/tests/scripts/compile_dep.sh/0.8.0/mogon2.out +++ b/tests/scripts/compile_dep.sh/0.8.0/mogon2.out @@ -3,11 +3,11 @@ ######## Installing: lz4 ############################### ######## Installing: snappy ############################### ######## Installing: capstone ############################### -######## Installing: libfabric:experimental ############################### +######## Installing: libfabric%experimental ############################### ######## Installing: mercury ############################### ######## Installing: argobots ############################### ######## Installing: margo ############################### -######## Installing: rocksdb:experimental ############################### +######## Installing: rocksdb%experimental ############################### ######## Installing: syscall_intercept ############################### ######## Installing: date ############################### ######## Installing: psm2 ############################### diff --git a/tests/scripts/compile_dep.sh/0.8.0/ngio.out b/tests/scripts/compile_dep.sh/0.8.0/ngio.out index a0f9661d4..ec0cb2c4d 100644 --- a/tests/scripts/compile_dep.sh/0.8.0/ngio.out +++ b/tests/scripts/compile_dep.sh/0.8.0/ngio.out @@ -2,7 +2,7 @@ ######## Installing: lz4 ############################### ######## Installing: snappy ############################### ######## Installing: capstone ############################### -######## Installing: libfabric:experimental ############################### +######## Installing: libfabric%experimental ############################### ######## Installing: mercury ############################### ######## Installing: argobots ############################### ######## Installing: margo ############################### diff --git a/tests/scripts/dl_dep.sh/0.8.0/default.out b/tests/scripts/dl_dep.sh/0.8.0/default.out index 2189a3b3b..bb81f52f9 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/default.out +++ b/tests/scripts/dl_dep.sh/0.8.0/default.out @@ -4,12 +4,12 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.8.1' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:verbs' with commit '[HEAD]' and flags '--branch=v1.7.2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.7.2' Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb:experimental' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' diff --git a/tests/scripts/dl_dep.sh/0.8.0/mogon1.out b/tests/scripts/dl_dep.sh/0.8.0/mogon1.out index c57e7b49a..3fe32ccb7 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/mogon1.out +++ b/tests/scripts/dl_dep.sh/0.8.0/mogon1.out @@ -2,7 +2,7 @@ Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:verbs' with commit '[HEAD]' and flags '--branch=v1.7.2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.7.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' diff --git a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out index 2d38d6bbe..a506c43fa 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out @@ -3,11 +3,11 @@ Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb:experimental' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental' Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' diff --git a/tests/scripts/dl_dep.sh/0.8.0/ngio.out b/tests/scripts/dl_dep.sh/0.8.0/ngio.out index 8bd3573aa..8885bc297 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/ngio.out +++ b/tests/scripts/dl_dep.sh/0.8.0/ngio.out @@ -2,7 +2,7 @@ Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' -Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric:experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' -- GitLab From 8a0867c87981d81714c9ac87dc19e7852ac0e5d2 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 25 Nov 2021 12:17:59 +0100 Subject: [PATCH 18/19] dep_scripts: Remove unused PATCH_DIR in compile_dep.sh --- scripts/compile_dep.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index ce17de93a..d02aa967c 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -28,7 +28,6 @@ ################################################################################ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PATCH_DIR="${PATCH_DIR}/patches" DEPENDENCY="" CORES="" SOURCE_DIR="" -- GitLab From 25fd2f35a83095eee7fbd7cfcecd17a5f5271e42 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 30 Nov 2021 11:21:51 +0100 Subject: [PATCH 19/19] dep_scripts: Allow install scripts to easily check the provided dependencies --- scripts/compile_dep.sh | 35 +++++++++++++++++++ .../profiles/0.8.0/install/mercury.install | 6 ++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index d02aa967c..cba79229b 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -330,6 +330,41 @@ determine_compiler() { fi } +# Check whether the loaded profile defines a particular dependency name. +# The function requires a valid bash regex argument to do the search. The +# function is meant to be used in a conditional context. +# +# Examples: +# 1. Check whether any flavor of 'libfabric' is defined by the profile: +# +# if profile_has_dependency "^libfabric%.*$"; then +# echo "libfabric found" +# fi +# +# 2. Check whether a specific flavor of 'libfabric' is defined by the profile: +# +# if profile_has_dependency "^libfabric%experimental$"; then +# echo "libfabric.experimental found" +# fi +profile_has_dependency() { + + if [[ "$#" -ne 1 ]]; then + >&2 echo "FATAL: Missing argument in profile_has_dependency()" + exit 1 + fi + + regex="$1" + + for name in "${PROFILE_DEP_LIST[@]}"; do + + if [[ "${name}" =~ ${regex} ]]; then + return 0 + fi + done + + return 1 +} + POSITIONAL=() while [[ $# -gt 0 ]]; do diff --git a/scripts/profiles/0.8.0/install/mercury.install b/scripts/profiles/0.8.0/install/mercury.install index 16376a66a..50394d861 100644 --- a/scripts/profiles/0.8.0/install/mercury.install +++ b/scripts/profiles/0.8.0/install/mercury.install @@ -45,8 +45,6 @@ pkg_install() { - set -x - # if the profile compiles bmi, enable it if [[ -n "${PROFILE_DEP_NAMES['bmi']}" ]]; then USE_BMI="-DNA_USE_BMI:BOOL=ON" @@ -54,8 +52,8 @@ pkg_install() { USE_BMI="-DNA_USE_BMI:BOOL=OFF" fi - # if the profile compiles libfabric, enable it - if [[ -n "${PROFILE_DEP_NAMES['libfabric']}" ]]; then + # if the profile provides any flavour of libfabric, enable it + if profile_has_dependency "^libfabric.*$"; then USE_OFI="-DNA_USE_OFI:BOOL=ON" else USE_OFI="-DNA_USE_OFI:BOOL=OFF" -- GitLab