script fixes (7c6421d8) · Commits · hpc / gekkofs

CMakeLists.txt

+5 −3

Original line number	Diff line number	Diff line
		@@ -65,6 +65,11 @@ set(CMAKE_C_FLAGS_MEMCHECK "${WARNINGS_FLAGS} -g -O0 -fsanitize=address -fno-omi
		set(CMAKE_C_FLAGS_MAINTAINER "${WARNINGS_FLAGS} -g -O0 -pg -no-pie")
		mark_as_advanced(CMAKE_CXX_FLAGS_MAINTAINER)

		# CMake and general includes
		include(CheckCXXSourceCompiles)
		include(CMakeDependentOption)
		include(GNUInstallDirs)

		# Project version
		set(GIT_VERSION_FOUND FALSE)
		if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/.git)
		@@ -245,8 +250,6 @@ include_directories(
		${CMAKE_BINARY_DIR}/include
		)

		include(GNUInstallDirs)

		# Common components
		add_subdirectory(src/common)
		# Daemon
		@@ -256,7 +259,6 @@ add_subdirectory(src/client)

		option(GKFS_BUILD_TESTS "Build GekkoFS self tests" OFF)

		include(CMakeDependentOption)
		cmake_dependent_option(GKFS_INSTALL_TESTS "Install GekkoFS self tests" OFF "GKFS_BUILD_TESTS" OFF)

		if (GKFS_BUILD_TESTS)

hermes @ 97820ed8

Original line number	Diff line number	Diff line
		Subproject commit 0af45bfa667f7ff9c78167ef94d975bffbd879f0
		Subproject commit 97820ed81aba9ef7c05891ce80c53a8533b78609

scripts/compile_dep.sh

+66 −16

Original line number	Diff line number	Diff line
		@@ -34,6 +34,8 @@ CORES=""
		SOURCE_DIR=""
		INSTALL_DIR=""
		PERFORM_TEST=
		VERBOSE=false
		LOG_FILE="/home/evie/remote_mac/gekkofs/scripts/my.log"

		DEFAULT_PROFILE="default"
		DEFAULT_VERSION="latest"
		@@ -45,6 +47,22 @@ declare -A PROFILE_DEP_NAMES
		declare -A PROFILE_WGETDEPS PROFILE_CLONEDEPS PROFILE_SOURCES PROFILE_EXTRA_INSTALL_ARGS
		declare -A PROFILE_CLONEDEPS_ARGS PROFILE_CLONEDEPS_PATCHES

		# colors and shell styles
		# check for color support and set variables
		if [ "$(tput colors)" -gt 2 ]; then
		C_GREEN='\033[0;32m' # green
		C_BYELLOW='\033[1;33m' # Bold yellow
		C_BRED='\033[1;31m' # Bold red
		C_NONE='\033[0m' # No color
		else
		C_GREEN=''
		C_BYELLOW=''
		C_BRED=''
		C_NONE=''
		fi
		# save stdout fd
		exec 5>&1

		usage_short() {
		echo "
		usage: compile_dep.sh [-h]
		@@ -89,9 +107,23 @@ optional arguments:
		number of cores that are used to compile the dependencies
		defaults to number of available cores
		-t, --test Perform libraries tests.
		-v, --verbose Increase download verbosity
		"
		}

		error_exit() {
		echo -e ">>> ${C_BRED}Failed${C_NONE} to $1: ${C_GREEN}$2${C_NONE}" >&5
		if [[ "$VERBOSE" != true ]]; then
		echo -e ">>> Log file: '${C_GREEN}${LOG_FILE}${C_NONE}'" >&5
		fi
		# print error info to shell
		dep_log_start=$(grep -m 1 -n "<installing $2>" ${LOG_FILE} \| cut -d : -f 1)
		log_size=$(wc -l ${LOG_FILE} \| cut -d " " -f 1)
		dep_log_lines=$((log_size-dep_log_start+1))
		[[ $dep_log_lines -gt 0 ]] && tail -n $dep_log_lines $LOG_FILE >&5
		exit 1
		}

		list_versions() {

		if [[ ! -d "${PROFILES_DIR}" ]]; then
		@@ -358,6 +390,10 @@ while [[ $# -gt 0 ]]; do
		exit
		#shift # past argument
		;;
		-v \| --verbose)
		VERBOSE=true
		shift # past argument
		;;
		*) # unknown option
		POSITIONAL+=("$1") # save it in an array for later
		shift # past argument
		@@ -379,15 +415,15 @@ INSTALL_DIR="$(readlink -mn "${2}")"
		# deal with optional arguments
		if [[ "${CORES}" == "" ]]; then
		CORES=$(grep -c ^processor /proc/cpuinfo)
		echo "CORES = ${CORES} (default)"
		echo -e ">>> Used CPU cores: ${C_GREEN}${CORES}${C_NONE} (default)"
		else
		if [[ ! "${CORES}" -gt "0" ]]; then
		echo "ERROR: CORES set to ${CORES} which is invalid.
		echo "${C_BRED}ERROR:${C_NONE} CORES set to ${C_GREEN}${CORES}${C_NONE} which is invalid.
		Input must be numeric and greater than 0."
		usage_short
		exit
		else
		echo CORES = "${CORES}"
		echo -e ">>> Used CPU cores: ${C_GREEN}${CORES}${C_NONE}"
		fi
		fi

		@@ -396,47 +432,61 @@ load_profile "${PROFILE_NAME}" "${PROFILE_VERSION}"
		CMAKE=$(find_cmake)
		CMAKE="${CMAKE} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}"

		echo "Sources download path = ${SOURCE_DIR}"
		echo "Installation path = ${INSTALL_DIR}"
		echo -e ">>> Log file: ${C_GREEN}${LOG_FILE}${C_NONE}"
		echo -e ">>> Profile set: ${C_GREEN}${PROFILE_NAME}:${PROFILE_VERSION}${C_NONE}"
		echo -e ">>> Sources download path set: ${C_GREEN}${SOURCE_DIR}${C_NONE}"
		echo -e ">>> Installation path set: ${C_GREEN}${INSTALL_DIR}${C_NONE}"
		echo "------------------------------------"

		mkdir -p "${SOURCE_DIR}"

		# truncate outfile
		: > /home/evie/remote_mac/gekkofs/scripts/my.log
		######### From now on exits on any error ########
		if [[ "$VERBOSE" == true ]]; then
		set -e
		else
		# from now on all stdout/stderr is going to the log file. Use fd 5 for stdout
		exec &>> $LOG_FILE
		set -e
		fi

		export CPATH="${CPATH}:${INSTALL_DIR}/include"
		export LIBRARY_PATH="${LIBRARY_PATH}:${INSTALL_DIR}/lib:${INSTALL_DIR}/lib64"
		export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH}"

		if [[ -n "${DEPENDENCY}" && ! -n "${PROFILE_DEP_NAMES[${DEPENDENCY}]}" ]]; then
		echo "Dependency '${DEPENDENCY}' not found in '${PROFILE_NAME}:${PROFILE_VERSION}'"
		echo "${C_BRED}ERROR:${C_NONE} Dependency '${DEPENDENCY}' not found in '${PROFILE_NAME}:${PROFILE_VERSION}'" >&5
		exit
		fi

		dep_cnt=0
		for dep in "${PROFILE_DEP_LIST[@]}"; do

		dep_cnt=$((dep_cnt+1))
		if [[ -n "${DEPENDENCY}" && "${dep}" != "${DEPENDENCY}" ]]; then
		continue
		fi

		install_script="${PROFILES_DIR}/${PROFILE_VERSION}/install/${dep}.install"

		echo -e "\n\n######## Installing: ${dep} ###############################\n"

		echo -e ">>> Building (${C_BYELLOW}${dep_cnt}${C_NONE} of ${C_BYELLOW}${#PROFILE_DEP_NAMES[@]}${C_NONE}) ${C_GREEN}${dep}${C_NONE}" >&5
		if [[ "$VERBOSE" != true ]]; then
		# Log message only
		echo -e "\n\n######## <installing ${dep}> ###############################\n"
		fi
		if [[ -f "${install_script}" ]]; then
		source "${install_script}"
		else
		echo "WARNING: Install script for '${dep}' not found. Skipping."
		echo "${C_BYELLOW}WARNING${C_NONE}: Install script for '${dep}' not found. Skipping." >&5
		continue
		fi
		pkg_install \|\| error_exit "build" "${dep}" $?

		pkg_install

		[ "${PERFORM_TEST}" ] && pkg_check
		if [ "${PERFORM_TEST}" ]; then
		pkg_check \|\| error_exit "test" "${dep}" $?
		fi

		done

		echo "Done"
		echo -e "\rJobs: ${C_BYELLOW}${dep_cnt}${C_NONE} of ${C_BYELLOW}${#PROFILE_DEP_NAMES[@]}${C_NONE} complete" >&5

		exit 0

scripts/dl_dep.sh

+43 −5

Original line number	Diff line number	Diff line
		@@ -33,6 +33,7 @@ PATCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
		PATCH_DIR="${PATCH_DIR}/patches"
		DEPENDENCY=""
		VERBOSE=false
		LOG_FILE="/home/evie/remote_mac/gekkofs/scripts/my.log"

		DEFAULT_PROFILE="default"
		DEFAULT_VERSION="latest"
		@@ -42,21 +43,43 @@ declare -a PROFILE_DEP_NAMES
		declare -A PROFILE_WGETDEPS PROFILE_CLONEDEPS PROFILE_SOURCES
		declare -A PROFILE_CLONEDEPS_ARGS PROFILE_CLONEDEPS_PATCHES

		# colors and shell styles
		# check for color support and set variables
		if [ "$(tput colors)" -gt 2 ]; then
		C_GREEN='\033[0;32m' # green
		C_BYELLOW='\033[1;33m' # Bold yellow
		C_BRED='\033[1;31m' # Bold red
		C_NONE='\033[0m' # No color
		else
		C_GREEN=''
		C_BYELLOW=''
		C_BRED=''
		C_NONE=''
		fi

		# Stop all backround jobs on interruption.
		# "kill -- -$$" sends a SIGTERM to the whole process group,
		# thus killing also descendants.
		# Use single quotes, otherwise this expands now rather than when signalled.
		# See shellcheck SC2064.
		trap 'trap - SIGTERM && kill -- -$$' SIGINT SIGTERM
		# save stdout fd
		exec 5>&1

		exit_child() {
		if [ ! $? -eq 0 ]; then
		# notify the parent
		kill -s SIGTERM -- -$$
		fi
		# progress loading bar
		printf "#" >&5
		}

		error_exit() {
		echo -e ">>> ${C_BRED}Failed${C_NONE} to download: ${C_GREEN}$2${C_NONE}" >&5
		if [[ "$VERBOSE" != true ]]; then
		echo -e ">>> Log file: '${C_GREEN}${LOG_FILE}${C_NONE}'" >&5
		fi
		echo "$1" >&2 ## Send message to stderr. Exclude >&2 if you don't want it that way.
		exit "${2:-1}" ## Return a code specified by $2 or 1 by default.
		}
		@@ -232,6 +255,7 @@ clonedeps() {
		if [[ "$VERBOSE" == true ]]; then
		set -ex
		else
		exec &>> $LOG_FILE
		set -e
		fi
		trap exit_child EXIT
		@@ -248,7 +272,7 @@ clonedeps() {
		cd "${SOURCE_DIR}/${FOLDER}" && git fetch -q
		ACTION="Pulled"
		else
		git clone ${COMMON_GIT_FLAGS} ${GIT_FLAGS} -- "${REPO}" "${SOURCE_DIR}/${FOLDER}"
		git clone ${COMMON_GIT_FLAGS} ${GIT_FLAGS} -- "${REPO}" "${SOURCE_DIR}/${FOLDER}" \|\| error_exit "Failed to clone ${REPO} ${GIT_FLAGS}" "${REPO}" $?
		ACTION="Cloned"
		fi
		# fix the version
		@@ -265,6 +289,7 @@ wgetdeps() {
		if [[ "$VERBOSE" == true ]]; then
		set -ex
		else
		exec &>> $LOG_FILE
		set -e
		fi
		trap exit_child EXIT
		@@ -281,7 +306,7 @@ wgetdeps() {
		if [[ -f "${SOURCE_DIR}/$FILENAME" ]]; then
		rm -f "${SOURCE_DIR}/$FILENAME"
		fi
		curl ${COMMON_CURL_FLAGS} "$URL" \|\| error_exit "Failed to download ${URL}" $?
		curl ${COMMON_CURL_FLAGS} "$URL" \|\| error_exit "Failed to download ${URL}" "${URL}" $?
		tar -xf "$FILENAME" --directory "${SOURCE_DIR}/${FOLDER}" --strip-components=1
		rm -f "$FILENAME"
		echo "Downloaded '${URL}' to '${FOLDER}'"
		@@ -402,14 +427,21 @@ if [[ -z ${1+x} ]]; then
		fi
		SOURCE_DIR="$(readlink -mn "${1}")"

		echo "Destination path is set to \"${SOURCE_DIR}\""
		echo -e ">>> Log file: ${C_GREEN}${LOG_FILE}${C_NONE}"
		echo -e ">>> Profile set: ${C_GREEN}${PROFILE_NAME}:${PROFILE_VERSION}${C_NONE}"
		echo -e ">>> Sources download path set: ${C_GREEN}${SOURCE_DIR}${C_NONE}"
		echo "------------------------------------"

		mkdir -p "${SOURCE_DIR}"

		# truncate outfile
		: > /home/evie/remote_mac/gekkofs/scripts/my.log

		dep_cnt=0
		loading_size=""
		## download dependencies
		for dep in "${PROFILE_DEP_NAMES[@]}"; do

		dep_cnt=$((dep_cnt+1))
		if [[ ! -z "${PROFILE_WGETDEPS[${dep}]:-}" ]]; then

		# dependency names can include a TAG after a colon (e.g. ofi:verbs),
		@@ -429,6 +461,7 @@ for dep in "${PROFILE_DEP_NAMES[@]}"; do

		dep_url="${dep_url/\{\{VERSION\}\}/${dep_version}}"

		echo -e ">>> Downloading (${C_BYELLOW}${dep_cnt}${C_NONE} of ${C_BYELLOW}${#PROFILE_DEP_NAMES[@]}${C_NONE}) ${C_GREEN}${dep}-${dep_version}${C_NONE}"
		wgetdeps "${dep_id}" "${dep_url}" &

		elif [[ ! -z "${PROFILE_CLONEDEPS[${dep}]:-}" ]]; then
		@@ -464,16 +497,21 @@ for dep in "${PROFILE_DEP_NAMES[@]}"; do

		dep_patch=${PROFILE_CLONEDEPS_PATCHES[${dep}]}

		echo -e ">>> Downloading (${C_BYELLOW}${dep_cnt}${C_NONE} of ${C_BYELLOW}${#PROFILE_DEP_NAMES[@]}${C_NONE}) ${C_GREEN}${dep}-${dep_version} ${dep_args}${C_NONE}"
		clonedeps "${dep}" "${dep_url}" "${dep_version}" "${dep_args}" "${dep_patch}" &

		else
		echo "Unknown dependency '${dep}'."
		exit 1
		fi
		loading_size="$loading_size "
		done

		# Wait for all download to be completed
		echo -e ">>> Waiting for jobs to finish ... "
		printf "%s]" "${loading_size}"
		printf "\r"
		wait
		echo "Done"
		echo -e "\rJobs: ${C_BYELLOW}${dep_cnt}${C_NONE} of ${C_BYELLOW}${#PROFILE_DEP_NAMES[@]}${C_NONE} complete"

		exit 0

scripts/profiles/0.8.0/default.specs

+40 −22

Original line number	Diff line number	Diff line
		@@ -37,31 +37,49 @@ comment="All dependencies"

		# Dependencies that must be downloaded directly
		wgetdeps=(
		["bzip2"]="1.0.6"
		["zstd"]="1.3.2"
		["lz4"]="1.8.0"
		["snappy"]="1.1.7"
		["capstone"]="4.0.1"
		["argobots"]="1.0rc1"
		["rocksdb"]="6.2.2"
		["rocksdb:experimental"]="6.11.4"
		["psm2"]="11.2.86"
		#master
		#["lz4"]="1.8.0"
		#["capstone"]="4.0.1"
		#["argobots"]="1.0rc1"
		#["rocksdb"]="6.2.2"
		# proxy_experiments
		#["lz4"]="1.8.0"
		#["capstone"]="4.0.1"
		#["argobots"]="1.1"
		#["rocksdb"]="6.16.4"
		# newest
		["lz4"]="1.9.3"
		["capstone"]="4.0.2"
		["argobots"]="1.1"
		["rocksdb"]="6.26.0"
		)

		# Dependencies that must be cloned
		clonedeps=(
		["bmi"]="6ea0b78fce1b964e45102828cdd05df7040a94c8"
		["libfabric"]="HEAD@v1.8.1"
		# ["libfabric:experimental"]="HEAD@v1.9.1"
		# ["libfabric:verbs"]="HEAD@v1.7.2"
		["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd"
		["margo"]="v0.6.3"
		["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd"
		#master
		#["libfabric"]="HEAD@v1.8.1"
		#["mercury"]="41caa143a07ed179a3149cac4af0dc7aa3f946fd"
		#["margo"]="v0.6.3"
		#["syscall_intercept"]="f7cebb7b7e7512a19b78a31ce236ad6ca22636dd"
		#["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
		#["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development"
		# proxy_experiments
		#["libfabric"]="HEAD@v1.12.1"
		#["mercury"]="4796aeeb90ba58be8a3b17a73f27aa3afed1ee0f"
		#["margo"]="34bff25aa8ec9e9348ee07a455308bfe3c17d623"
		#["syscall_intercept"]="304404581c57d43478438d175099d20260bae74e"
		#["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
		#["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development"
		# newest
		["libfabric"]="HEAD@v1.13.2"
		["mercury"]="HEAD@v2.1.0rc3"
		["margo"]="HEAD@v0.9.6"
		["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d"
		["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155"
		["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development"
		)

		# Extra arguments for git clone
		# Extra arguments for git clone TODO no space is added automatically
		clonedeps_args=(
		["mercury"]=" --recurse-submodules"
		)
		@@ -73,12 +91,12 @@ clonedeps_patches=(

		# Ordering that MUST be followed when downloading
		order=(
		"bzip2" "zstd" "lz4" "snappy" "capstone" "bmi"
		"lz4" "capstone"
		"libfabric"
		#"libfabric:experimental"
		#"libfabric:verbs"
		"mercury" "argobots" "margo" "rocksdb" "rocksdb:experimental"
		"syscall_intercept" "date" "psm2" "agios"
		"mercury" "argobots" "margo" "rocksdb"
		"syscall_intercept" "date" "agios"
		)

		# Extra arguments passed to the installation script. As such, they can
		@@ -90,7 +108,7 @@ order=(
		# - CORES: the number of cores to use when building
		# - PERFORM_TEST: whether tests for the package should be executed
		extra_install_args=(
		["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2"
		#["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2"
		#["libfabric:verbs"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes"
		#["libfabric:experimental"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2 --enable-verbs=yes"
		)