From f097f50566332dcc874426670fc970f44c0807fd Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 15 Oct 2020 05:45:33 +0000 Subject: [PATCH 1/3] Download and compile AGIOS in the CI. --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d846bcd7f..13a129a64 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,7 +37,9 @@ compile dependencies: - ( [ -f "${DEPS_COMMIT}" ] && git diff --quiet "`cat ${DEPS_COMMIT}`" -- scripts/dl_dep.sh scripts/compile_dep.sh ) || ( rm -f ${DEPS_COMMIT} && scripts/dl_dep.sh ${DEPS_SRC_PATH} -n ofi && + scripts/dl_dep.sh ${DEPS_SRC_PATH} -n ofi -d agios && scripts/compile_dep.sh -n ofi ${DEPS_SRC_PATH} ${DEPS_INSTALL_PATH} && + scripts/compile_dep.sh -n ofi -d agios ${DEPS_SRC_PATH} ${DEPS_INSTALL_PATH} && echo "${CI_COMMIT_SHA}" > "${DEPS_COMMIT}" ) artifacts: -- GitLab From 618148ba8125689f901d4dbef77d8296c3731b91 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 15 Oct 2020 05:58:31 +0000 Subject: [PATCH 2/3] Updated dl, comp scripts and ci to download AGIOS --- .gitlab-ci.yml | 6 ++---- scripts/compile_dep.sh | 15 ++++++++++++++- scripts/dl_dep.sh | 17 +++++++++++++++-- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 13a129a64..62ed47b31 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,10 +36,8 @@ compile dependencies: # If the download and compile script have been modified the cache gets invalidated and dependencies will be built again. - ( [ -f "${DEPS_COMMIT}" ] && git diff --quiet "`cat ${DEPS_COMMIT}`" -- scripts/dl_dep.sh scripts/compile_dep.sh ) || ( rm -f ${DEPS_COMMIT} && - scripts/dl_dep.sh ${DEPS_SRC_PATH} -n ofi && - scripts/dl_dep.sh ${DEPS_SRC_PATH} -n ofi -d agios && - scripts/compile_dep.sh -n ofi ${DEPS_SRC_PATH} ${DEPS_INSTALL_PATH} && - scripts/compile_dep.sh -n ofi -d agios ${DEPS_SRC_PATH} ${DEPS_INSTALL_PATH} && + scripts/dl_dep.sh ${DEPS_SRC_PATH} -n ofi -c ci && + scripts/compile_dep.sh -n ofi -c ci ${DEPS_SRC_PATH} ${DEPS_INSTALL_PATH} && echo "${CI_COMMIT_SHA}" > "${DEPS_COMMIT}" ) artifacts: diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index f15ff66c5..da2dd50b2 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -35,6 +35,10 @@ ALL_DEPS=( "syscall_intercept" "date" "agios" ) +CI_DEPS=( + "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" +) + usage_short() { echo " usage: compile_dep.sh [-h] [-l] [-n ] [-c ] [-d ] [-j ] @@ -62,7 +66,7 @@ optional arguments: defaults to 'all' -c , --config allows additional configurations, e.g., for specific clusters - supported values: {mogon1, mogon2, ngio, direct, all} + supported values: {mogon1, mogon2, ngio, direct, all, ci} defaults to 'direct' -d , --dependency download a specific dependency and ignore --config setting. If unspecified @@ -104,6 +108,11 @@ list_dependencies() { echo -n "$d " done echo + echo -n " ci: " + for d in "${CI_DEPS[@]}"; do + echo -n "$d " + done + echo } check_dependency() { @@ -254,6 +263,10 @@ all) DEP_CONFIG=("${ALL_DEPS[@]}") echo "'All' dependencies are compiled" ;; +ci) + DEP_CONFIG=("${CI_DEPS[@]}") + echo "'CI' dependencies are compiled" + ;; direct | *) DEP_CONFIG=("${DIRECT_DEPS[@]}") echo "'Direct' GekkoFS dependencies are compiled (default)" diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index d299330f2..53ddcc77a 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -35,6 +35,10 @@ ALL_DEPS=( "syscall_intercept" "date" "agios" ) +CI_DEPS=( + "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" +) + # Stop all backround jobs on interruption. # "kill -- -$$" sends a SIGTERM to the whole process group, # thus killing also descendants. @@ -82,7 +86,12 @@ list_dependencies() { for d in "${ALL_DEPS[@]}"; do echo -n "$d " done - echo + echo + echo -n " ci: " + for d in "${CI_DEPS[@]}"; do + echo -n "$d " + done + echo } check_dependency() { @@ -191,7 +200,7 @@ optional arguments: defaults to 'ofi' -c , --config allows additional configurations, e.g., for specific clusters - supported values: {mogon2, mogon1, ngio, direct, all} + supported values: {mogon2, mogon1, ngio, direct, all, ci} defaults to 'direct' -d , --dependency download a specific dependency and ignore --config setting. If unspecified @@ -285,6 +294,10 @@ all) DEP_CONFIG=("${ALL_DEPS[@]}") [[ -z "${DEPENDENCY}" ]] && echo "'All' dependencies are downloaded" ;; +ci) + DEP_CONFIG=("${CI_DEPS[@]}") + [[ -z "${DEPENDENCY}" ]] && echo "'CI' dependencies are downloaded" + ;; direct | *) DEP_CONFIG=("${DIRECT_DEPS[@]}") [[ -z "${DEPENDENCY}" ]] && echo "'Direct' GekkoFS dependencies are downloaded (default)" -- GitLab From 80a80fbb0aa73ac24bcd67949e3f1104a5ae11c6 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 15 Oct 2020 06:00:24 +0000 Subject: [PATCH 3/3] Update scripts/dl_dep.sh, scripts/compile_dep.sh files --- scripts/compile_dep.sh | 2 +- scripts/dl_dep.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index da2dd50b2..a7d35ebe6 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -9,7 +9,7 @@ SOURCE="" INSTALL="" DEP_CONFIG="" -VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all" +VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all ci" MOGON1_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index 53ddcc77a..8d2616916 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -9,7 +9,7 @@ NA_LAYER="" DEP_CONFIG="" VERBOSE=false -VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all" +VALID_DEP_OPTIONS="mogon2 mogon1 ngio direct all ci" MOGON1_DEPS=( "zstd" "lz4" "snappy" "capstone" "ofi-verbs" "mercury" "argobots" "margo" "rocksdb" -- GitLab