Loading scripts/compile_dep.sh +243 −208 Original line number Original line Diff line number Diff line #!/bin/bash #!/bin/bash MOGON1_DEPS=( PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" "zstd" "lz4" "snappy" "bmi" "libfabric" "mercury" "argobots" "margo" PATCH_DIR="${PATCH_DIR}/patches" "rocksdb" "syscall_intercept date" DEPENDENCY="" ) NA_LAYER="" CORES="" SOURCE="" INSTALL="" DEP_CONFIG="" VALID_DEP_OPTIONS="mogon2 direct all" MOGON2_DEPS=( MOGON2_DEPS=( "zstd" "lz4" "snappy" "bmi" "mercury" "argobots" "margo" "rocksdb" "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept date" "syscall_intercept" "date" ) DIRECT_DEPS=( "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" ) ) FH2_DEPS=( ALL_DEPS=( "zstd" "lz4" "snappy" "bmi" "libfabric" "mercury" "argobots" "margo" "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb" "rocksdb" "syscall_intercept date" "syscall_intercept" "date" ) ) usage_short() { usage_short() { echo " echo " usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CLUSTER>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>] usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CONFIG>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>] source_path install_path source_path install_path " " } } Loading @@ -40,12 +50,14 @@ optional arguments: -n <NAPLUGIN>, --na <NAPLUGIN> -n <NAPLUGIN>, --na <NAPLUGIN> network layer that is used for communication. Valid: {bmi,ofi,all} network layer that is used for communication. Valid: {bmi,ofi,all} defaults to 'all' defaults to 'all' -c <CLUSTER>, --cluster <CLUSTER> -c <CONFIG>, --config <CONFIG> additional configurations for specific compute clusters allows additional configurations, e.g., for specific clusters supported clusters: {mogon1,mogon2,fh2} supported values: {mogon2, direct, all} defaults to 'direct' -d <DEPENDENCY>, --dependency <DEPENDENCY> -d <DEPENDENCY>, --dependency <DEPENDENCY> build and install a specific dependency. If unspecified download a specific dependency and ignore --config setting. If unspecified all dependencies are built and installed. all dependencies are built and installed based on set --config setting. Multiple dependencies are supported: Pass a space-separated string (e.g., \"ofi mercury\" -j <COMPILE_CORES>, --compilecores <COMPILE_CORES> -j <COMPILE_CORES>, --compilecores <COMPILE_CORES> number of cores that are used to compile the dependencies number of cores that are used to compile the dependencies defaults to number of available cores defaults to number of available cores Loading @@ -53,61 +65,66 @@ optional arguments: " " } } list_dependencies() { list_dependencies() { echo "Available dependencies: " echo "Available dependencies: " echo -n " Mogon 1: " echo -n " Mogon 2: " for d in "${MOGON1_DEPS[@]}" for d in "${MOGON2_DEPS[@]}"; do do echo -n "$d " echo -n "$d " done done echo "" echo -n " Direct GekkoFS dependencies: " for d in "${DIRECT_DEPS[@]}"; do echo -n " Mogon 2: " for d in "${MOGON2_DEPS[@]}" do echo -n "$d " echo -n "$d " done done echo "" echo -n " All: " for d in "${ALL_DEPS[@]}"; do echo -n " fh2: " for d in "${FH2_DEPS[@]}" do echo -n "$d " echo -n "$d " done done echo "" echo "" } } check_dependency() { local DEP=$1 shift local DEP_CONFIG=("$@") # ignore template when specific dependency is set if [[ -n "${DEPENDENCY}" ]]; then # check if specific dependency was set and return from function if echo "${DEPENDENCY}" | grep -q "${DEP}"; then return fi # [[ "${DEPENDENCY}" == "${DEP}" ]] && return else # if not check if dependency is part of dependency config for e in "${DEP_CONFIG[@]}"; do if [[ "${DEP}" == "${e}" ]]; then return fi done fi false } prepare_build_dir() { prepare_build_dir() { if [ ! -d "$1/build" ]; then if [ ! -d "$1/build" ]; then mkdir $1/build mkdir "$1"/build fi fi rm -rf $1/build/* rm -rf "$1"/build/* } } find_cmake() { find_cmake() { local CMAKE=`command -v cmake3 || command -v cmake` local CMAKE CMAKE=$(command -v cmake3 || command -v cmake) if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then >&2 echo "ERROR: could not find cmake" echo >&2 "ERROR: could not find cmake" exit 1 exit 1 fi fi echo ${CMAKE} echo "${CMAKE}" } } PATCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" PATCH_DIR="${PATCH_DIR}/patches" CLUSTER="" DEPENDENCY="" NA_LAYER="" CORES="" SOURCE="" INSTALL="" POSITIONAL=() POSITIONAL=() while [[ $# -gt 0 ]] while [[ $# -gt 0 ]]; do do key="$1" key="$1" case ${key} in case ${key} in Loading @@ -116,14 +133,22 @@ case ${key} in shift # past argument shift # past argument shift # past value shift # past value ;; ;; -c|--cluster) -c | --config) CLUSTER="$2" if [[ -z "$2" ]]; then echo "ERROR: Missing argument for -c/--config option" exit 1 fi if ! echo "$VALID_DEP_OPTIONS" | grep -q "$2"; then echo "ERROR: Invalid argument for -c/--config option" exit 1 fi TMP_DEP_CONF="$2" shift # past argument shift # past argument shift # past value shift # past value ;; ;; -d | --dependency) -d | --dependency) if [[ -z "$2" ]]; then if [[ -z "$2" ]]; then echo "Missing argument for -d/--dependency option" echo "ERROR: Missing argument for -d/--dependency option" exit exit fi fi DEPENDENCY="$2" DEPENDENCY="$2" Loading Loading @@ -158,7 +183,7 @@ set -- "${POSITIONAL[@]}" # restore positional parameters # deal with positional arguments # deal with positional arguments if [[ (-z ${1+x}) || (-z ${2+x}) ]]; then if [[ (-z ${1+x}) || (-z ${2+x}) ]]; then echo "Positional arguments missing." echo "ERROR: Positional arguments missing." usage_short usage_short exit 1 exit 1 fi fi Loading @@ -166,16 +191,16 @@ SOURCE="$( readlink -mn "${1}" )" INSTALL="$(readlink -mn "${2}")" INSTALL="$(readlink -mn "${2}")" # deal with optional arguments # deal with optional arguments if [ "${NA_LAYER}" == "" ]; then if [[ "${NA_LAYER}" == "" ]]; then echo "Defaulting NAPLUGIN to 'all'" echo "Defaulting NAPLUGIN to 'ofi'" NA_LAYER="all" NA_LAYER="ofi" fi fi if [ "${CORES}" == "" ]; then if [[ "${CORES}" == "" ]]; then CORES=$(grep -c ^processor /proc/cpuinfo) CORES=$(grep -c ^processor /proc/cpuinfo) echo "CORES = ${CORES} (default)" echo "CORES = ${CORES} (default)" else else if [ ! "${CORES}" -gt "0" ]; then if [[ ! "${CORES}" -gt "0" ]]; then echo "CORES set to ${CORES} which is invalid. echo "ERROR: CORES set to ${CORES} which is invalid. Input must be numeric and greater than 0." Input must be numeric and greater than 0." usage_short usage_short exit exit Loading @@ -183,35 +208,40 @@ Input must be numeric and greater than 0." echo CORES = "${CORES}" echo CORES = "${CORES}" fi fi fi fi if [ "${NA_LAYER}" == "bmi" ] || [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then echo NAPLUGIN = "${NA_LAYER}" echo NAPLUGIN = "${NA_LAYER}" else else echo "No valid plugin selected" echo "ERROR: No valid NA plugin selected" usage_short usage_short exit exit fi fi if [[ "${CLUSTER}" != "" ]]; then # enable predefined dependency template if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) || ( "${CLUSTER}" == "mogon2" ) ]]; then case ${TMP_DEP_CONF} in echo CLUSTER = "${CLUSTER}" mogon2) else DEP_CONFIG=("${MOGON2_DEPS[@]}") echo "${CLUSTER} cluster configuration is invalid. Exiting ..." echo "'Mogon2' dependencies are compiled" usage_short ;; exit all) fi DEP_CONFIG=("${ALL_DEPS[@]}") else echo "'All' dependencies are compiled" echo "No cluster configuration set." ;; fi direct | *) DEP_CONFIG=("${DIRECT_DEPS[@]}") echo "'Direct' GekkoFS dependencies are compiled (default)" ;; esac USE_BMI="-DNA_USE_BMI:BOOL=OFF" USE_BMI="-DNA_USE_BMI:BOOL=OFF" USE_OFI="-DNA_USE_OFI:BOOL=OFF" USE_OFI="-DNA_USE_OFI:BOOL=OFF" CMAKE=`find_cmake` CMAKE=$(find_cmake) CMAKE="${CMAKE} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}" CMAKE="${CMAKE} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}" echo "Source path = ${SOURCE}"; echo "Source path = ${SOURCE}" echo "Install path = ${INSTALL}"; echo "Install path = ${INSTALL}" echo "------------------------------------" mkdir -p ${SOURCE} mkdir -p "${SOURCE}" ######### From now on exits on any error ######## ######### From now on exits on any error ######## set -e set -e Loading @@ -219,64 +249,72 @@ set -e export CPATH="${CPATH}:${INSTALL}/include" export CPATH="${CPATH}:${INSTALL}/include" export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL}/lib:${INSTALL}/lib64" export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL}/lib:${INSTALL}/lib64" ## Third party dependencies # Set cluster dependencies first # Set cluster dependencies first if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) || ( "${CLUSTER}" == "mogon2" ) ]]; then # compile zstd # build zstd for fast compression in rocksdb if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "zstd" ) ]]; then if check_dependency "zstd" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: zstd" echo "############################################################ Installing: zstd" CURR=${SOURCE}/zstd/build/cmake CURR=${SOURCE}/zstd/build/cmake prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR}/build cd "${CURR}"/build $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Release .. $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Release .. make -j${CORES} make -j"${CORES}" make install make install fi fi # build lz4 # build zlib for rocksdb if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "zstd" ) ]]; then if check_dependency "lz4" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: zstd" echo "############################################################ Installing: lz4" CURR=${SOURCE}/lz4 CURR=${SOURCE}/lz4 cd ${CURR} cd "${CURR}" make -j${CORES} make -j"${CORES}" make DESTDIR=${INSTALL} PREFIX="" install make DESTDIR="${INSTALL}" PREFIX="" install fi fi # build snappy # build snappy for rocksdb if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "snappy" ) ]]; then if check_dependency "snappy" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: snappy" echo "############################################################ Installing: snappy" CURR=${SOURCE}/snappy CURR=${SOURCE}/snappy prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR}/build cd "${CURR}"/build $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Release .. $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Release .. make -j${CORES} make -j"${CORES}" make install make install fi fi # build capstone for syscall-intercept if check_dependency "capstone" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: capstone" CURR=${SOURCE}/capstone prepare_build_dir "${CURR}" cd "${CURR}"/build $CMAKE -DCMAKE_INSTALL_PREFIX=/home/vef/gekkofs_deps/install -DCMAKE_BUILD_TYPE:STRING=Release .. make -j"${CORES}" install fi fi # build bmi # build BMI if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "bmi" ) ]]; then if check_dependency "bmi" "${DEP_CONFIG[@]}"; then if [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "all" ]; then if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "all" ]]; then USE_BMI="-DNA_USE_BMI:BOOL=ON" USE_BMI="-DNA_USE_BMI:BOOL=ON" echo "############################################################ Installing: BMI" echo "############################################################ Installing: BMI" # BMI # BMI CURR=${SOURCE}/bmi CURR=${SOURCE}/bmi prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR} cd "${CURR}" ./prepare ./prepare cd ${CURR}/build cd "${CURR}"/build CFLAGS="${CFLAGS} -w" ../configure --prefix=${INSTALL} --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict CFLAGS="${CFLAGS} -w" ../configure --prefix="${INSTALL}" --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict make -j${CORES} make -j"${CORES}" make install make install fi fi fi fi # build ofi # build ofi if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "ofi" ) ]]; then if check_dependency "ofi" "${DEP_CONFIG[@]}"; then if [ "$NA_LAYER" == "ofi" ] || [ "$NA_LAYER" == "all" ]; then if [[ "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then USE_OFI="-DNA_USE_OFI:BOOL=ON" USE_OFI="-DNA_USE_OFI:BOOL=ON" # Mogon2 already has libfabric installed in a version that Mercury supports. if [[ ("${CLUSTER}" != "mogon2") ]]; then echo "############################################################ Installing: LibFabric" echo "############################################################ Installing: LibFabric" #libfabric #libfabric CURR=${SOURCE}/libfabric CURR=${SOURCE}/libfabric Loading @@ -288,24 +326,22 @@ if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "ofi" ) ]]; then [ "${PERFORM_TEST}" ] && make check [ "${PERFORM_TEST}" ] && make check fi fi fi fi fi # Mercury # Mercury if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "mercury" ) ]]; then if check_dependency "mercury" "${DEP_CONFIG[@]}"; then if [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "all" ]; then if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "all" ]]; then USE_BMI="-DNA_USE_BMI:BOOL=ON" USE_BMI="-DNA_USE_BMI:BOOL=ON" fi fi if [ "$NA_LAYER" == "ofi" ] || [ "$NA_LAYER" == "all" ]; then if [[ "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then USE_OFI="-DNA_USE_OFI:BOOL=ON" USE_OFI="-DNA_USE_OFI:BOOL=ON" fi fi echo "############################################################ Installing: Mercury" echo "############################################################ Installing: Mercury" CURR=${SOURCE}/mercury CURR=${SOURCE}/mercury prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR}/build cd "${CURR}"/build PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig $CMAKE \ PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig $CMAKE \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DBUILD_TESTING:BOOL=ON \ -DBUILD_TESTING:BOOL=ON \ Loading @@ -318,67 +354,66 @@ if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "mercury" ) ]]; then -DCMAKE_INSTALL_PREFIX=${INSTALL} \ -DCMAKE_INSTALL_PREFIX=${INSTALL} \ ${USE_BMI} ${USE_OFI} \ ${USE_BMI} ${USE_OFI} \ .. .. make -j${CORES} make -j"${CORES}" make install make install fi fi # Argobots # Argobots if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "argobots" ) ]]; then if check_dependency "argobots" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: Argobots" echo "############################################################ Installing: Argobots" CURR=${SOURCE}/argobots CURR=${SOURCE}/argobots prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR} cd "${CURR}" ./autogen.sh ./autogen.sh cd ${CURR}/build cd "${CURR}"/build ../configure --prefix=${INSTALL} --enable-perf-opt --disable-checks ../configure --prefix="${INSTALL}" --enable-perf-opt --disable-checks make -j${CORES} make -j"${CORES}" make install make install [ "${PERFORM_TEST}" ] && make check [ "${PERFORM_TEST}" ] && make check fi fi # Margo # Margo if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "margo" ) ]]; then if check_dependency "margo" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: Margo" echo "############################################################ Installing: Margo" CURR=${SOURCE}/margo CURR=${SOURCE}/margo prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR} cd "${CURR}" ./prepare.sh ./prepare.sh cd ${CURR}/build cd "${CURR}"/build ../configure --prefix=${INSTALL} PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig CFLAGS="${CFLAGS} -Wall -O3" ../configure --prefix="${INSTALL}" PKG_CONFIG_PATH="${INSTALL}"/lib/pkgconfig CFLAGS="${CFLAGS} -Wall -O3" make -j${CORES} make -j"${CORES}" make install make install [ "${PERFORM_TEST}" ] && make check [ "${PERFORM_TEST}" ] && make check fi fi # Rocksdb # Rocksdb if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "rocksdb" ) ]]; then if check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: Rocksdb" echo "############################################################ Installing: Rocksdb" CURR=${SOURCE}/rocksdb CURR=${SOURCE}/rocksdb cd ${CURR} cd "${CURR}" make clean make clean USE_RTTI=1 make -j${CORES} static_lib USE_RTTI=1 make -j"${CORES}" static_lib INSTALL_PATH=${INSTALL} make install INSTALL_PATH="${INSTALL}" make install fi fi # syscall_intercept # syscall_intercept if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "syscall_intercept" ) ]]; then if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: Syscall_intercept" echo "############################################################ Installing: Syscall_intercept" CURR=${SOURCE}/syscall_intercept CURR=${SOURCE}/syscall_intercept prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR}/build cd "${CURR}"/build $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF .. $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF .. make install make install fi fi # date # date if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "date" ) ]]; then if check_dependency "date" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: date" echo "############################################################ Installing: date" CURR=${SOURCE}/date CURR=${SOURCE}/date prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR}/build cd "${CURR}"/build $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON .. $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON .. make install make install fi fi echo "Done" echo "Done" scripts/dl_dep.sh +179 −138 File changed.Preview size limit exceeded, changes collapsed. Show changes Loading
scripts/compile_dep.sh +243 −208 Original line number Original line Diff line number Diff line #!/bin/bash #!/bin/bash MOGON1_DEPS=( PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" "zstd" "lz4" "snappy" "bmi" "libfabric" "mercury" "argobots" "margo" PATCH_DIR="${PATCH_DIR}/patches" "rocksdb" "syscall_intercept date" DEPENDENCY="" ) NA_LAYER="" CORES="" SOURCE="" INSTALL="" DEP_CONFIG="" VALID_DEP_OPTIONS="mogon2 direct all" MOGON2_DEPS=( MOGON2_DEPS=( "zstd" "lz4" "snappy" "bmi" "mercury" "argobots" "margo" "rocksdb" "zstd" "lz4" "snappy" "capstone" "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept date" "syscall_intercept" "date" ) DIRECT_DEPS=( "ofi" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" ) ) FH2_DEPS=( ALL_DEPS=( "zstd" "lz4" "snappy" "bmi" "libfabric" "mercury" "argobots" "margo" "zstd" "lz4" "snappy" "capstone" "bmi" "ofi" "mercury" "argobots" "margo" "rocksdb" "rocksdb" "syscall_intercept date" "syscall_intercept" "date" ) ) usage_short() { usage_short() { echo " echo " usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CLUSTER>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>] usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CONFIG>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>] source_path install_path source_path install_path " " } } Loading @@ -40,12 +50,14 @@ optional arguments: -n <NAPLUGIN>, --na <NAPLUGIN> -n <NAPLUGIN>, --na <NAPLUGIN> network layer that is used for communication. Valid: {bmi,ofi,all} network layer that is used for communication. Valid: {bmi,ofi,all} defaults to 'all' defaults to 'all' -c <CLUSTER>, --cluster <CLUSTER> -c <CONFIG>, --config <CONFIG> additional configurations for specific compute clusters allows additional configurations, e.g., for specific clusters supported clusters: {mogon1,mogon2,fh2} supported values: {mogon2, direct, all} defaults to 'direct' -d <DEPENDENCY>, --dependency <DEPENDENCY> -d <DEPENDENCY>, --dependency <DEPENDENCY> build and install a specific dependency. If unspecified download a specific dependency and ignore --config setting. If unspecified all dependencies are built and installed. all dependencies are built and installed based on set --config setting. Multiple dependencies are supported: Pass a space-separated string (e.g., \"ofi mercury\" -j <COMPILE_CORES>, --compilecores <COMPILE_CORES> -j <COMPILE_CORES>, --compilecores <COMPILE_CORES> number of cores that are used to compile the dependencies number of cores that are used to compile the dependencies defaults to number of available cores defaults to number of available cores Loading @@ -53,61 +65,66 @@ optional arguments: " " } } list_dependencies() { list_dependencies() { echo "Available dependencies: " echo "Available dependencies: " echo -n " Mogon 1: " echo -n " Mogon 2: " for d in "${MOGON1_DEPS[@]}" for d in "${MOGON2_DEPS[@]}"; do do echo -n "$d " echo -n "$d " done done echo "" echo -n " Direct GekkoFS dependencies: " for d in "${DIRECT_DEPS[@]}"; do echo -n " Mogon 2: " for d in "${MOGON2_DEPS[@]}" do echo -n "$d " echo -n "$d " done done echo "" echo -n " All: " for d in "${ALL_DEPS[@]}"; do echo -n " fh2: " for d in "${FH2_DEPS[@]}" do echo -n "$d " echo -n "$d " done done echo "" echo "" } } check_dependency() { local DEP=$1 shift local DEP_CONFIG=("$@") # ignore template when specific dependency is set if [[ -n "${DEPENDENCY}" ]]; then # check if specific dependency was set and return from function if echo "${DEPENDENCY}" | grep -q "${DEP}"; then return fi # [[ "${DEPENDENCY}" == "${DEP}" ]] && return else # if not check if dependency is part of dependency config for e in "${DEP_CONFIG[@]}"; do if [[ "${DEP}" == "${e}" ]]; then return fi done fi false } prepare_build_dir() { prepare_build_dir() { if [ ! -d "$1/build" ]; then if [ ! -d "$1/build" ]; then mkdir $1/build mkdir "$1"/build fi fi rm -rf $1/build/* rm -rf "$1"/build/* } } find_cmake() { find_cmake() { local CMAKE=`command -v cmake3 || command -v cmake` local CMAKE CMAKE=$(command -v cmake3 || command -v cmake) if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then >&2 echo "ERROR: could not find cmake" echo >&2 "ERROR: could not find cmake" exit 1 exit 1 fi fi echo ${CMAKE} echo "${CMAKE}" } } PATCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" PATCH_DIR="${PATCH_DIR}/patches" CLUSTER="" DEPENDENCY="" NA_LAYER="" CORES="" SOURCE="" INSTALL="" POSITIONAL=() POSITIONAL=() while [[ $# -gt 0 ]] while [[ $# -gt 0 ]]; do do key="$1" key="$1" case ${key} in case ${key} in Loading @@ -116,14 +133,22 @@ case ${key} in shift # past argument shift # past argument shift # past value shift # past value ;; ;; -c|--cluster) -c | --config) CLUSTER="$2" if [[ -z "$2" ]]; then echo "ERROR: Missing argument for -c/--config option" exit 1 fi if ! echo "$VALID_DEP_OPTIONS" | grep -q "$2"; then echo "ERROR: Invalid argument for -c/--config option" exit 1 fi TMP_DEP_CONF="$2" shift # past argument shift # past argument shift # past value shift # past value ;; ;; -d | --dependency) -d | --dependency) if [[ -z "$2" ]]; then if [[ -z "$2" ]]; then echo "Missing argument for -d/--dependency option" echo "ERROR: Missing argument for -d/--dependency option" exit exit fi fi DEPENDENCY="$2" DEPENDENCY="$2" Loading Loading @@ -158,7 +183,7 @@ set -- "${POSITIONAL[@]}" # restore positional parameters # deal with positional arguments # deal with positional arguments if [[ (-z ${1+x}) || (-z ${2+x}) ]]; then if [[ (-z ${1+x}) || (-z ${2+x}) ]]; then echo "Positional arguments missing." echo "ERROR: Positional arguments missing." usage_short usage_short exit 1 exit 1 fi fi Loading @@ -166,16 +191,16 @@ SOURCE="$( readlink -mn "${1}" )" INSTALL="$(readlink -mn "${2}")" INSTALL="$(readlink -mn "${2}")" # deal with optional arguments # deal with optional arguments if [ "${NA_LAYER}" == "" ]; then if [[ "${NA_LAYER}" == "" ]]; then echo "Defaulting NAPLUGIN to 'all'" echo "Defaulting NAPLUGIN to 'ofi'" NA_LAYER="all" NA_LAYER="ofi" fi fi if [ "${CORES}" == "" ]; then if [[ "${CORES}" == "" ]]; then CORES=$(grep -c ^processor /proc/cpuinfo) CORES=$(grep -c ^processor /proc/cpuinfo) echo "CORES = ${CORES} (default)" echo "CORES = ${CORES} (default)" else else if [ ! "${CORES}" -gt "0" ]; then if [[ ! "${CORES}" -gt "0" ]]; then echo "CORES set to ${CORES} which is invalid. echo "ERROR: CORES set to ${CORES} which is invalid. Input must be numeric and greater than 0." Input must be numeric and greater than 0." usage_short usage_short exit exit Loading @@ -183,35 +208,40 @@ Input must be numeric and greater than 0." echo CORES = "${CORES}" echo CORES = "${CORES}" fi fi fi fi if [ "${NA_LAYER}" == "bmi" ] || [ "${NA_LAYER}" == "ofi" ] || [ "${NA_LAYER}" == "all" ]; then if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then echo NAPLUGIN = "${NA_LAYER}" echo NAPLUGIN = "${NA_LAYER}" else else echo "No valid plugin selected" echo "ERROR: No valid NA plugin selected" usage_short usage_short exit exit fi fi if [[ "${CLUSTER}" != "" ]]; then # enable predefined dependency template if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) || ( "${CLUSTER}" == "mogon2" ) ]]; then case ${TMP_DEP_CONF} in echo CLUSTER = "${CLUSTER}" mogon2) else DEP_CONFIG=("${MOGON2_DEPS[@]}") echo "${CLUSTER} cluster configuration is invalid. Exiting ..." echo "'Mogon2' dependencies are compiled" usage_short ;; exit all) fi DEP_CONFIG=("${ALL_DEPS[@]}") else echo "'All' dependencies are compiled" echo "No cluster configuration set." ;; fi direct | *) DEP_CONFIG=("${DIRECT_DEPS[@]}") echo "'Direct' GekkoFS dependencies are compiled (default)" ;; esac USE_BMI="-DNA_USE_BMI:BOOL=OFF" USE_BMI="-DNA_USE_BMI:BOOL=OFF" USE_OFI="-DNA_USE_OFI:BOOL=OFF" USE_OFI="-DNA_USE_OFI:BOOL=OFF" CMAKE=`find_cmake` CMAKE=$(find_cmake) CMAKE="${CMAKE} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}" CMAKE="${CMAKE} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}" echo "Source path = ${SOURCE}"; echo "Source path = ${SOURCE}" echo "Install path = ${INSTALL}"; echo "Install path = ${INSTALL}" echo "------------------------------------" mkdir -p ${SOURCE} mkdir -p "${SOURCE}" ######### From now on exits on any error ######## ######### From now on exits on any error ######## set -e set -e Loading @@ -219,64 +249,72 @@ set -e export CPATH="${CPATH}:${INSTALL}/include" export CPATH="${CPATH}:${INSTALL}/include" export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL}/lib:${INSTALL}/lib64" export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL}/lib:${INSTALL}/lib64" ## Third party dependencies # Set cluster dependencies first # Set cluster dependencies first if [[ ( "${CLUSTER}" == "mogon1" ) || ( "${CLUSTER}" == "fh2" ) || ( "${CLUSTER}" == "mogon2" ) ]]; then # compile zstd # build zstd for fast compression in rocksdb if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "zstd" ) ]]; then if check_dependency "zstd" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: zstd" echo "############################################################ Installing: zstd" CURR=${SOURCE}/zstd/build/cmake CURR=${SOURCE}/zstd/build/cmake prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR}/build cd "${CURR}"/build $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Release .. $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Release .. make -j${CORES} make -j"${CORES}" make install make install fi fi # build lz4 # build zlib for rocksdb if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "zstd" ) ]]; then if check_dependency "lz4" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: zstd" echo "############################################################ Installing: lz4" CURR=${SOURCE}/lz4 CURR=${SOURCE}/lz4 cd ${CURR} cd "${CURR}" make -j${CORES} make -j"${CORES}" make DESTDIR=${INSTALL} PREFIX="" install make DESTDIR="${INSTALL}" PREFIX="" install fi fi # build snappy # build snappy for rocksdb if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "snappy" ) ]]; then if check_dependency "snappy" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: snappy" echo "############################################################ Installing: snappy" CURR=${SOURCE}/snappy CURR=${SOURCE}/snappy prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR}/build cd "${CURR}"/build $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Release .. $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Release .. make -j${CORES} make -j"${CORES}" make install make install fi fi # build capstone for syscall-intercept if check_dependency "capstone" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: capstone" CURR=${SOURCE}/capstone prepare_build_dir "${CURR}" cd "${CURR}"/build $CMAKE -DCMAKE_INSTALL_PREFIX=/home/vef/gekkofs_deps/install -DCMAKE_BUILD_TYPE:STRING=Release .. make -j"${CORES}" install fi fi # build bmi # build BMI if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "bmi" ) ]]; then if check_dependency "bmi" "${DEP_CONFIG[@]}"; then if [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "all" ]; then if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "all" ]]; then USE_BMI="-DNA_USE_BMI:BOOL=ON" USE_BMI="-DNA_USE_BMI:BOOL=ON" echo "############################################################ Installing: BMI" echo "############################################################ Installing: BMI" # BMI # BMI CURR=${SOURCE}/bmi CURR=${SOURCE}/bmi prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR} cd "${CURR}" ./prepare ./prepare cd ${CURR}/build cd "${CURR}"/build CFLAGS="${CFLAGS} -w" ../configure --prefix=${INSTALL} --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict CFLAGS="${CFLAGS} -w" ../configure --prefix="${INSTALL}" --enable-shared --disable-static --disable-karma --enable-bmi-only --enable-fast --disable-strict make -j${CORES} make -j"${CORES}" make install make install fi fi fi fi # build ofi # build ofi if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "ofi" ) ]]; then if check_dependency "ofi" "${DEP_CONFIG[@]}"; then if [ "$NA_LAYER" == "ofi" ] || [ "$NA_LAYER" == "all" ]; then if [[ "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then USE_OFI="-DNA_USE_OFI:BOOL=ON" USE_OFI="-DNA_USE_OFI:BOOL=ON" # Mogon2 already has libfabric installed in a version that Mercury supports. if [[ ("${CLUSTER}" != "mogon2") ]]; then echo "############################################################ Installing: LibFabric" echo "############################################################ Installing: LibFabric" #libfabric #libfabric CURR=${SOURCE}/libfabric CURR=${SOURCE}/libfabric Loading @@ -288,24 +326,22 @@ if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "ofi" ) ]]; then [ "${PERFORM_TEST}" ] && make check [ "${PERFORM_TEST}" ] && make check fi fi fi fi fi # Mercury # Mercury if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "mercury" ) ]]; then if check_dependency "mercury" "${DEP_CONFIG[@]}"; then if [ "$NA_LAYER" == "bmi" ] || [ "$NA_LAYER" == "all" ]; then if [[ "${NA_LAYER}" == "bmi" || "${NA_LAYER}" == "all" ]]; then USE_BMI="-DNA_USE_BMI:BOOL=ON" USE_BMI="-DNA_USE_BMI:BOOL=ON" fi fi if [ "$NA_LAYER" == "ofi" ] || [ "$NA_LAYER" == "all" ]; then if [[ "${NA_LAYER}" == "ofi" || "${NA_LAYER}" == "all" ]]; then USE_OFI="-DNA_USE_OFI:BOOL=ON" USE_OFI="-DNA_USE_OFI:BOOL=ON" fi fi echo "############################################################ Installing: Mercury" echo "############################################################ Installing: Mercury" CURR=${SOURCE}/mercury CURR=${SOURCE}/mercury prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR}/build cd "${CURR}"/build PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig $CMAKE \ PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig $CMAKE \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DBUILD_TESTING:BOOL=ON \ -DBUILD_TESTING:BOOL=ON \ Loading @@ -318,67 +354,66 @@ if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "mercury" ) ]]; then -DCMAKE_INSTALL_PREFIX=${INSTALL} \ -DCMAKE_INSTALL_PREFIX=${INSTALL} \ ${USE_BMI} ${USE_OFI} \ ${USE_BMI} ${USE_OFI} \ .. .. make -j${CORES} make -j"${CORES}" make install make install fi fi # Argobots # Argobots if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "argobots" ) ]]; then if check_dependency "argobots" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: Argobots" echo "############################################################ Installing: Argobots" CURR=${SOURCE}/argobots CURR=${SOURCE}/argobots prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR} cd "${CURR}" ./autogen.sh ./autogen.sh cd ${CURR}/build cd "${CURR}"/build ../configure --prefix=${INSTALL} --enable-perf-opt --disable-checks ../configure --prefix="${INSTALL}" --enable-perf-opt --disable-checks make -j${CORES} make -j"${CORES}" make install make install [ "${PERFORM_TEST}" ] && make check [ "${PERFORM_TEST}" ] && make check fi fi # Margo # Margo if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "margo" ) ]]; then if check_dependency "margo" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: Margo" echo "############################################################ Installing: Margo" CURR=${SOURCE}/margo CURR=${SOURCE}/margo prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR} cd "${CURR}" ./prepare.sh ./prepare.sh cd ${CURR}/build cd "${CURR}"/build ../configure --prefix=${INSTALL} PKG_CONFIG_PATH=${INSTALL}/lib/pkgconfig CFLAGS="${CFLAGS} -Wall -O3" ../configure --prefix="${INSTALL}" PKG_CONFIG_PATH="${INSTALL}"/lib/pkgconfig CFLAGS="${CFLAGS} -Wall -O3" make -j${CORES} make -j"${CORES}" make install make install [ "${PERFORM_TEST}" ] && make check [ "${PERFORM_TEST}" ] && make check fi fi # Rocksdb # Rocksdb if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "rocksdb" ) ]]; then if check_dependency "rocksdb" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: Rocksdb" echo "############################################################ Installing: Rocksdb" CURR=${SOURCE}/rocksdb CURR=${SOURCE}/rocksdb cd ${CURR} cd "${CURR}" make clean make clean USE_RTTI=1 make -j${CORES} static_lib USE_RTTI=1 make -j"${CORES}" static_lib INSTALL_PATH=${INSTALL} make install INSTALL_PATH="${INSTALL}" make install fi fi # syscall_intercept # syscall_intercept if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "syscall_intercept" ) ]]; then if check_dependency "syscall_intercept" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: Syscall_intercept" echo "############################################################ Installing: Syscall_intercept" CURR=${SOURCE}/syscall_intercept CURR=${SOURCE}/syscall_intercept prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR}/build cd "${CURR}"/build $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF .. $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTS:BOOK=OFF .. make install make install fi fi # date # date if [[ ( "${DEPENDENCY}" == "" ) || ( "${DEPENDENCY}" == "date" ) ]]; then if check_dependency "date" "${DEP_CONFIG[@]}"; then echo "############################################################ Installing: date" echo "############################################################ Installing: date" CURR=${SOURCE}/date CURR=${SOURCE}/date prepare_build_dir ${CURR} prepare_build_dir "${CURR}" cd ${CURR}/build cd "${CURR}"/build $CMAKE -DCMAKE_INSTALL_PREFIX=${INSTALL} -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON .. $CMAKE -DCMAKE_INSTALL_PREFIX="${INSTALL}" -DCMAKE_CXX_STANDARD:STRING=14 -DUSE_SYSTEM_TZ_DB:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON .. make install make install fi fi echo "Done" echo "Done"
scripts/dl_dep.sh +179 −138 File changed.Preview size limit exceeded, changes collapsed. Show changes