#!/bin/bash
# colors and shell styles
# check for color support and set variables
if [ "$(tput colors)" -gt 2 ]; then
    C_GREEN='\033[0;32m' # green
    C_BYELLOW='\033[1;33m' # Bold yellow
    C_BRED='\033[1;31m' # Bold red
    C_NONE='\033[0m' # No color
else
    C_GREEN=''
    C_BYELLOW=''
    C_BRED=''
    C_NONE=''
fi
C_AST_GREEN="${C_GREEN}*${C_NONE} [gkfs] "
C_AST_YELLOW="${C_BYELLOW}*${C_NONE} [gkfs] "
C_AST_RED="${C_BRED}*${C_NONE} [gkfs] "

count_host_lines() {
    local file="$1"
    grep -v '^#' "${file}" 2>/dev/null | grep -cv '^[[:space:]]*$'
}

count_before_mutate_host_lines() {
    local file="$1"
    grep -v '^#' "${file}" 2>/dev/null | grep -v '^[[:space:]]*+' | grep -cv '^[[:space:]]*$'
}

count_ready_daemons() {
    local directory="${HOSTSFILE}.ready"
    if [[ ! -d ${directory} ]]; then
        echo 0
        return
    fi
    find "${directory}" -maxdepth 1 -type f -name 'daemon_*.status' -printf '.' 2>/dev/null | wc -c
}

ensure_malleability_bin() {
    if [[ -n ${GKFS_MALLEABILITY_BIN_} ]]; then
        GKFS_MALLEABILITY_BIN_=$(readlink -f "${GKFS_MALLEABILITY_BIN_}")
    else
        GKFS_MALLEABILITY_BIN_=$(command -v gkfs_malleability)
    fi
    if [[ -z ${GKFS_MALLEABILITY_BIN_} ]]; then
        if [[ -f $(dirname "${DAEMON_BIN}")/gkfs_malleability ]]; then
            GKFS_MALLEABILITY_BIN_=$(readlink -f "$(dirname "${DAEMON_BIN}")/gkfs_malleability")
        elif [[ -f $(dirname "$(dirname "${DAEMON_BIN}")")/tools/gkfs_malleability ]]; then
            GKFS_MALLEABILITY_BIN_=$(readlink -f "$(dirname "$(dirname "${DAEMON_BIN}")")/tools/gkfs_malleability")
        else
            echo -e "${C_AST_RED}ERROR: gkfs_malleability binary not found. Exiting ..."
            exit 1
        fi
    fi
}

run_malleability_workflow() {
    local label="$1"
    local old_count="$2"
    ensure_malleability_bin
    export LIBGKFS_HOSTS_FILE=${HOSTSFILE}

    ${GKFS_MALLEABILITY_BIN_} mutate start
    echo -e "${C_AST_GREEN}${label} progress: "
    until MALLEABILITY_STATUS=$(${GKFS_MALLEABILITY_BIN_} --machine-readable mutate status 2>&1); [ $((${MALLEABILITY_STATUS})) -eq 0 ]
    do
        sleep 1
        show_expand_progress ${MALLEABILITY_STATUS} ${old_count}
    done
    show_expand_progress ${MALLEABILITY_STATUS} ${old_count}
    echo
    echo -e "${C_AST_GREEN}Redistribution process done. Finalizing ..."
    MALLEABILITY_FINALIZE=$(${GKFS_MALLEABILITY_BIN_} --machine-readable mutate finalize 2>&1)
    if [ $((${MALLEABILITY_FINALIZE})) -ne 0 ]; then
        echo -e "${C_AST_RED}ERROR: ${label} finalize failed. This is not recoverable. Exiting ..."
        exit 1
    fi
    echo -e "${C_AST_GREEN}${label} done."
}
#######################################
# Poll GekkoFS hostsfile until all daemons are started.
# Exits with 1 if daemons cannot be started.
# Globals:
#   HOSTSFILE
#   NODE_NUM
#   NODE_CNT_EXPAND
#   COMMAND
# Arguments:
#   None
# Outputs:
#   Writes error to stdout
#######################################
wait_for_gkfs_daemons() {
	  sleep 1
    local server_wait_cnt=0
    local nodes=1
    if [[ -n ${NODE_NUM} ]]; then
        nodes=${NODE_NUM}
    fi
    # when expanding/mutating with new daemons, the hostfile contains old entries
    # plus '+' entries written by the added daemons
    if [[ ${GKFS_DAEMON_EXPAND} == "ON" ]]; then
        nodes=${NODE_CNT_EXPAND}
    fi
    until [ $(($(count_host_lines "${HOSTSFILE}" | awk '{print $1}') + 0)) -eq "${nodes}" ] && \
          [ $(($(count_ready_daemons) + 0)) -eq "${nodes}" ]
    do
		    #echo "Waiting for all servers to report connection. Try $server_wait_cnt"
        sleep 2
        server_wait_cnt=$((server_wait_cnt+1))
        if [ ${server_wait_cnt} -gt 600 ]; then
            echo -e "${C_AST_RED}Server failed to start. Exiting ..."
            exit 1
        fi
    done
}
#######################################
# Creates a pid file for a given pid. If pid file exists, we check if its pids are still valid.
# If valid, an additional line is added. Otherwise, the pid in the file is deleted.
# Globals:
#   DAEMON_PID_FILE
#   PROXY_PID_FILE
#   VERBOSE
# Arguments:
#   path to pid file
#   pid to write to pid file
# Outputs:
#   Writes status to stdout if VERBOSE is true
#######################################
write_pid_file() {
    local pid_file=${1}
    local pid=${2}
    if [[ ${VERBOSE} == true ]]; then
        echo -e "${C_AST_GREEN}Creating pid file at ${pid_file} with pid ${pid} ..."
    fi
    # if PID file exists another daemon (or srun) could run
    if [[ -e ${pid_file} ]]; then
        local pid_file_tmp=${DAEMON_PID_FILE}.swp
        # create empty tmp file
        truncate -s 0 "${pid_file_tmp}"
        while IFS= read -r line
        do
            if ps -p "${line}" > /dev/null; then
                # process with pid still running
                echo "${line}" >> "${pid_file_tmp}"
            fi
        done < "${pid_file}"
        # create pid file with only valid pids
        mv "${pid_file_tmp}" "${pid_file}"
    fi
    echo "${pid}" >> "${pid_file}"
}
#######################################
# Starts GekkoFS daemons.
# Globals:
#   SLURM_ACCOUNT
#   SLURM_PARTITION
#   SLURM_JOB_ID
#   NODE_NUM
#   MOUNTDIR
#   ROOTDIR
#   DAEMON_ARGS_
#   PROXY_ARGS_
#   CPUS_PER_TASK
#   VERBOSE
#   DAEMON_AFFINITY_
#   PROXY_AFFINITY_
#   USE_PROXY
#   DAEMON_CPUNODEBIND
#   DAEMON_MEMBIND
#   GKFS_DAEMON_LOG_PATH
#   GKFS_DAEMON_LOG_LEVEL
#   RUN_FOREGROUND
#   DAEMON_BIN
#   PROXY_BIN
#   COMMAND
# Outputs:
#   Writes status to stdout
#######################################
start_daemons() {
    local node_list
    local srun_daemon_cmd
    local srun_proxy_cmd
    local daemon_execute
    local proxy_execute
		# only used when started without slurm job
		local srun_account_partition
    # setup
    if [[ ${USE_SRUN} == true ]]; then
        if [[ -z ${SLURM_JOB_ID} ]]; then
            if [[ -z ${SLURM_ACCOUNT} || -z ${SLURM_PARTITION} ]]; then
                echo -e "${C_AST_RED}ERROR: Slurm Account or Partition not set. Exiting ..."
                exit 1
            fi
            srun_account_partition="--partition=${SLURM_PARTITION} --account=${SLURM_ACCOUNT}"
						# TODO remove
						echo -e "observe: ${srun_account_partition}"
        else
            srun_account_partition=""
        fi
        # check for daemon first
        if [[ -n ${DAEMON_NODELIST_} ]]; then
            if [[ ! -f ${DAEMON_NODELIST_} ]]; then
                echo -e "${C_AST_RED}ERROR: Daemon nodelist file not found at ${DAEMON_NODELIST_}. Exiting ..."
                exit 1
            fi
            NODE_NUM=$(wc -l < "${DAEMON_NODELIST_}")
            srun_daemon_cmd="srun --disable-status --nodelist=${DAEMON_NODELIST_} --ntasks=${NODE_NUM} --cpus-per-task=${CPUS_PER_TASK} ${srun_account_partition} ${SRUN_ARGS} "
        else
            node_list=$(scontrol show job "${SLURM_JOB_ID}" | grep " NodeList=" | cut -d "=" -f2)
            if [[ -z ${NODE_NUM} ]]; then
                NODE_NUM=$(scontrol show hostname "${node_list}" | wc -l)
            fi
            # Setting up base srun cmd
            srun_daemon_cmd="srun --disable-status -N ${NODE_NUM} --ntasks=${NODE_NUM} --cpus-per-task=${CPUS_PER_TASK} ${srun_account_partition} ${SRUN_ARGS} "
        fi
        if [[ ${USE_PROXY} == true ]]; then
            if [[ -n ${PROXY_NODELIST_} ]]; then
                if [[ ! -f ${PROXY_NODELIST_} ]]; then
                    echo -e "${C_AST_RED}ERROR: Proxy nodelist file not found at ${PROXY_NODELIST_}. Exiting ..."
                    exit 1
                fi
                NODE_NUM_PROXY=$(wc -l < "${PROXY_NODELIST_}")
                srun_proxy_cmd="srun --disable-status --nodelist=${PROXY_NODELIST_} --ntasks=${NODE_NUM_PROXY} --cpus-per-task=${CPUS_PER_TASK} ${srun_account_partition} ${SRUN_ARGS} "
            else
                srun_proxy_cmd="srun --disable-status -N ${NODE_NUM} --ntasks=${NODE_NUM} --cpus-per-task=${CPUS_PER_TASK} ${srun_account_partition} ${SRUN_ARGS} "
                NODE_NUM_PROXY=$NODE_NUM
            fi
        fi
    else
        NODE_NUM=1
        if [[ ${USE_PROXY} == true ]]; then
            NODE_NUM_PROXY=$NODE_NUM
        fi
    fi

    if [[ ${VERBOSE} == true ]]; then
        echo -e "${C_AST_GREEN}-------------------CLIENT SETTINGS------------------------"
        echo -e "${C_AST_GREEN}Set env variables for client: "
        echo -e "LD_PRELOAD=${PRELOAD_LIB}"
        echo -e "LIBGKFS_HOSTS_FILE=${HOSTSFILE}"
        echo -e "LIBGKFS_PROXY_PID_FILE=${PROXY_LOCAL_PID_FILE}"
        echo -e "${C_AST_GREEN}----------------------------------------------------------"
        echo -e "${C_AST_GREEN}mountdir: ${MOUNTDIR}"
        echo -e "${C_AST_GREEN}rootdir: ${ROOTDIR}"
        echo -e "${C_AST_GREEN}node_num: ${NODE_NUM}"
        echo -e "${C_AST_GREEN}additional daemon args: ${DAEMON_ARGS_}"
        echo -e "${C_AST_GREEN}cpus_per_task: ${CPUS_PER_TASK}"
        [[ ${USE_PROXY} == true ]] && echo -e "${C_AST_GREEN}Proxy enabled"
    fi
    # sanity checks before starting
    if [[ ${COMMAND} == *"start"* ]]; then
        # only clear hostfile when starting for the first time
        if [[ ${VERBOSE} == true ]]; then
            echo -e "${C_AST_GREEN}Cleaning host file ..."
        fi
        rm "${HOSTSFILE}" 2> /dev/null
    fi
    # Setting up base daemon cmd
    local daemon_cmd="${DAEMON_BIN} -r ${ROOTDIR} -m ${MOUNTDIR} -H ${HOSTSFILE} ${DAEMON_ARGS_}"
    if [[ ${USE_PROXY} == true ]]; then
        daemon_cmd="${daemon_cmd} ${DAEMON_PROXY_ARGS}"
    fi
    # Set cpu affinity for daemon
    if [[ -n ${DAEMON_AFFINITY_} ]]; then
        daemon_cmd="${DAEMON_AFFINITY_} ${daemon_cmd}"
    fi

    # add --time parameter to ARGS
    if [[ -n ${SRUN_TIME} ]]; then
      SRUN_DAEMON_ARGS="--time=${SRUN_TIME} ${SRUN_DAEMON_ARGS}"
      SRUN_PROXY_ARGS="--time=${SRUN_TIME} ${SRUN_PROXY_ARGS}"
    else
      SRUN_DAEMON_ARGS="--time=00:15:00 ${SRUN_DAEMON_ARGS}"
      SRUN_PROXY_ARGS="--time=00:15:00 ${SRUN_PROXY_ARGS}"
    fi

    # final daemon execute COMMAND
    daemon_execute="${srun_daemon_cmd} ${SRUN_DAEMON_ARGS} ${daemon_cmd}"

    # Setting up base proxy COMMAND
    if [[ ${USE_PROXY} == true ]]; then
        local proxy_cmd="${PROXY_BIN} -H ${HOSTSFILE} --pid-path ${PROXY_LOCAL_PID_FILE} ${PROXY_ARGS_}"
        # Set cpu affinity for proxy
        if [[ -n ${PROXY_AFFINITY_} ]]; then
            proxy_cmd="${PROXY_AFFINITY_} ${proxy_cmd}"
        fi
        # final proxy execute COMMAND
        proxy_execute="${srun_proxy_cmd} ${SRUN_PROXY_ARGS} ${proxy_cmd}"
    fi

    if [[ ${VERBOSE} == true ]]; then
        echo -e "${C_AST_GREEN}Full execute DAEMON COMMAND:"
        echo -e "${C_AST_GREEN}# $daemon_execute"
        [[ ${USE_PROXY} == true ]] && echo -e "${C_AST_GREEN}Full execute PROXY COMMAND:"
        [[ ${USE_PROXY} == true ]] && echo -e "${C_AST_GREEN}# $proxy_execute"
    fi
    # setup environment variables
    export GKFS_DAEMON_LOG_PATH=$GKFS_DAEMON_LOG_PATH
    export GKFS_DAEMON_LOG_LEVEL=$GKFS_DAEMON_LOG_LEVEL
    export GKFS_PROXY_LOG_PATH=$GKFS_PROXY_LOG_PATH
    export GKFS_PROXY_LOG_LEVEL=$GKFS_PROXY_LOG_LEVEL

    echo -e "${C_AST_GREEN}Starting GekkoFS daemons (${NODE_NUM} nodes) ..."
    start_time="$(date -u +%s.%3N)"
    ${daemon_execute} &
    local daemon_pid=$!
    wait_for_gkfs_daemons
    stop_time="$(date -u +%s.%3N)"
    elapsed="$(bc <<<"$stop_time-$start_time")"
    echo -e "${C_AST_GREEN}GekkoFS daemons running"
    echo -e "${C_AST_GREEN}Startup time: ${elapsed} seconds"

    if [[ ${USE_PROXY} == true ]]; then
        echo -e "${C_AST_GREEN}Starting GekkoFS proxies (${NODE_NUM_PROXY} nodes) ..."
        start_time="$(date -u +%s.%3N)"
        ${proxy_execute} &
        local proxy_pid=$!
        sleep 5 # TODO
        stop_time="$(date -u +%s.%3N)"
        elapsed="$(bc <<<"$stop_time-$start_time")"
        echo -e "${C_AST_GREEN}GekkoFS proxies probably :) running"
        echo -e "${C_AST_GREEN}Startup time: ${elapsed} seconds"
    fi

    if [[ ${RUN_FOREGROUND} == true ]]; then
        echo "Press 'q' to exit"
        while : ; do
            read -n 1 k <&1
            if [[ $k = q ]] ; then
                if [[ ${USE_PROXY} == true ]]; then
                    start_time="$(date -u +%s.%3N)"
                    echo
                    echo -e "${C_AST_GREEN}Shutting down GekkoFS proxies ..."
                    if [[ -n ${proxy_pid} ]]; then
                        kill -s SIGINT ${proxy_pid} &
                        wait ${proxy_pid}
                    fi
                    stop_time="$(date -u +%s.%3N)"
                    elapsed="$(bc <<<"$stop_time-$start_time")"
                    echo -e "${C_AST_GREEN}Shutdown time: ${elapsed} seconds"
                fi
                start_time="$(date -u +%s.%3N)"
                echo
                echo -e "${C_AST_GREEN}Shutting down GekkoFS daemons ..."
                if [[ -n ${daemon_pid} ]]; then
                    kill -s SIGINT ${daemon_pid} &
                    wait ${daemon_pid}
                fi
                stop_time="$(date -u +%s.%3N)"
                elapsed="$(bc <<<"$stop_time-$start_time")"
                echo -e "${C_AST_GREEN}Shutdown time: ${elapsed} seconds"
                break
            else
                echo "Press 'q' to exit"
            fi
        done
    else
        write_pid_file ${DAEMON_PID_FILE} ${daemon_pid}
        if [[ ${USE_PROXY} == true ]]; then
            write_pid_file ${PROXY_PID_FILE} ${proxy_pid}
        fi
    fi
}
#######################################
# Stops GekkoFS daemons for the configured pid file
# Globals:
#   DAEMON_PID_FILE
#   PROXY_PID_FILE
#   VERBOSE
# Outputs:
#   Writes status to stdout
#######################################
stop_daemons() {
    local pid_file=${DAEMON_PID_FILE}
    local proxy_pid_file=${PROXY_PID_FILE}
    # if no daemon or proxy pid file exists, exit
    if [[ ! -e ${pid_file} ]] && [[ ! -e ${proxy_pid_file} ]]; then
        echo -e "${C_AST_RED}No pid files found -> no daemon or proxy running. Exiting ..."
        exit 1
    fi
    # attempt to shutdown proxy
    if [[ ${USE_PROXY} == true ]] && [[ -e ${proxy_pid_file} ]]; then
        while IFS= read -r line
        do
            if ps -p "${line}" > /dev/null; then
                echo -e "${C_AST_GREEN}Stopping proxy with pid ${line}"
                start_time="$(date -u +%s.%3N)"
                kill -s SIGINT "${line}" &
                # poll pid until it stopped
                if [[ ${VERBOSE} == true ]]; then
                    echo -e "${C_AST_GREEN}Waiting for proxies to exit ..."
                fi
                timeout 1 tail --pid=${line} -f /dev/null
            fi
        done < "${proxy_pid_file}"
        rm "${proxy_pid_file}"
        stop_time="$(date -u +%s.%3N)"
        elapsed="$(bc <<<"$stop_time-$start_time")"
        echo -e "${C_AST_GREEN}Shutdown time: ${elapsed} seconds"
    fi
    # attempt to shutdown daemon
    if [[ -e ${pid_file} ]]; then
        while IFS= read -r line
        do
            # if line starts with # continue
            [[ ${line} =~ ^#.*$ ]] && continue
            if ps -p "${line}" > /dev/null; then
                echo -e "${C_AST_GREEN}Stopping daemon with pid ${line}"
                start_time="$(date -u +%s.%3N)"
                kill -s SIGINT "${line}" &
                # poll pid until it stopped
                if [[ ${VERBOSE} == true ]]; then
                    echo -e "${C_AST_GREEN}Waiting for daemons to exit ..."
                fi
                timeout 1 tail --pid=${line} -f /dev/null
            fi
        done < "${pid_file}"
        rm "${pid_file}"
        stop_time="$(date -u +%s.%3N)"
        elapsed="$(bc <<<"$stop_time-$start_time")"
        echo -e "${C_AST_GREEN}Shutdown time: ${elapsed} seconds"
    fi
}

#######################################
# Prints expansion progress
# Input:
#   $1 current
#   $2 total
#   VERBOSE
# Outputs:
#   Writes status to stdout
#######################################
show_expand_progress() {
    local current="$1"
    local total="$2"
    local remaining=$((total - current))
    local progress=$(( (remaining * 100) / total ))
    local bar_length=20
    local filled_length=$(( (progress * bar_length) / 100 ))
    local empty_length=$(( bar_length - filled_length ))

    # Clear the entire line and move cursor to the beginning
    tput el1; tput cr

    printf "["
    for ((i=0; i<filled_length; i++)); do
        printf "#"
    done
    for ((i=0; i<empty_length; i++)); do
        printf " "
    done

    printf "] %d/%d left" "$current" "$total"
}

mutate_daemons() {
    if [[ ${RUN_FOREGROUND} == true ]]; then
        echo -e "${C_AST_RED}ERROR: Cannot run in foreground for mutate. Exiting ..."
        exit 1
    fi
    if [[ ${USE_PROXY} == true ]]; then
        echo -e "${C_AST_RED}ERROR: Proxy not supported for mutate. Exiting ..."
        exit 1
    fi
    if [[ ! -f ${HOSTSFILE} ]]; then
        echo -e "${C_AST_RED}ERROR: No GekkoFS hostfile found at ${HOSTSFILE}. Exiting ..."
        exit 1
    fi
    local node_cnt_initial=$(count_before_mutate_host_lines "${HOSTSFILE}")
    run_malleability_workflow "Mutate" "${node_cnt_initial}"
}

malleability_status() {
    ensure_malleability_bin
    export LIBGKFS_HOSTS_FILE=${HOSTSFILE}
    ${GKFS_MALLEABILITY_BIN_} mutate status
}

malleability_finalize() {
    ensure_malleability_bin
    export LIBGKFS_HOSTS_FILE=${HOSTSFILE}
    ${GKFS_MALLEABILITY_BIN_} mutate finalize
}

#######################################
# Print short usage information
# Outputs:
#   Writes help to stdout
#######################################
usage_short() {
    echo "
usage: gkfs [-h/--help] [-r/--rootdir <path>] [-m/--mountdir <path>] [-d/--daemon_args <daemon_args>] [--proxy <false>] [-f/--foreground <false>]
        [--srun <false>] [-n/--numnodes <jobsize>] [--cpuspertask <64>] [-H/--hostsfile <path>] [-v/--verbose <false>]
        {start,mutate,status,finalize,stop}
    "
}
#######################################
# Print detailed usage information
# Outputs:
#   Writes help to stdout
#######################################
help_msg() {
    usage_short
    echo "
    This script simplifies the starting and stopping GekkoFS daemons. If daemons are started on multiple nodes,
    a Slurm environment is required. The script looks for the 'gkfs.conf' file in the same directory where
    additional permanent configurations can be set.

    positional arguments:
            COMMAND                 Command to execute: 'start', 'stop', 'mutate', 'status', 'finalize'

    optional arguments:
            -h, --help              Shows this help message and exits
            -r, --rootdir <path>    The rootdir path for GekkoFS daemons.
            -m, --mountdir <path>   The mountdir path for GekkoFS daemons.
            -d, --daemon_args <daemon_arguments>
            --proxy                 Start proxy after the daemons are running.
                                    Add various additional daemon arguments, e.g., \"-l ib0 -P ofi+psm2\".
            -p, --proxy_args <proxy_arguments>
            -f, --foreground        Starts the script in the foreground. Daemons are stopped by pressing 'q'.
            --srun                  Use srun to start daemons on multiple nodes.
            -n, --numnodes <n>      GekkoFS daemons are started on n nodes.
                                    Nodelist is extracted from Slurm via the SLURM_JOB_ID env variable.
            --cpuspertask <#cores>  Set the number of cores the daemons can use. Must use '--srun'.
            -c, --config            Path to configuration file. By defaults looks for a 'gkfs.conf' in this directory.
            -H, --hostsfile <path>  Path to the GekkoFS hostfile. For mutate, this may already contain '+' and '-' markers.
            -v, --verbose           Increase verbosity
            -t, --time              Set a limit on the total run time of the slurm job allocation.
            -A, --account           Account for the slurm job (only required for job allocation)
            -P, --partition         Partition for the slurm job (only required for job allocation)
    malleability examples:
            gkfs mutate
            "
}
CONFIGPATH=""
argv=("$@")
# get config path first from argument list
for i in "${!argv[@]}"; do
    if [[ "${argv[i]}" == "-c" || "${argv[i]}" == "--config" ]]; then
        CONFIGPATH=$(readlink -mn "${argv[i+1]}")
        break
    fi
done
# global variables
export FI_PSM2_DISCONNECT=1
export PSM2_MULTI_EP=1
SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"

# get default path if config wasn't given
if [[ -z ${CONFIGPATH} ]]; then
    CONFIGPATH="${SCRIPTDIR}/gkfs.conf"
fi
if [[ ! -f ${CONFIGPATH} ]]; then
    >&2 echo -e "${C_AST_RED}>> No config file found at '${CONFIGPATH}'."
    exit 1
fi
# get variables from CONFIGPATH
source "$CONFIGPATH"
# more global variables which may be overwritten by user input
VERBOSE=false
NODE_NUM=""
MOUNTDIR=${DAEMON_MOUNTDIR}
ROOTDIR=${DAEMON_ROOTDIR}
HOSTSFILE=${LIBGKFS_HOSTS_FILE}
CPUS_PER_TASK=$(grep -c ^processor /proc/cpuinfo)
DAEMON_ARGS_=${DAEMON_ARGS}
PROXY_ARGS_=${PROXY_ARGS}
USE_SRUN=${USE_SRUN}
DAEMON_NODELIST_=${DAEMON_NODELIST}
PROXY_NODELIST_=${PROXY_NODELIST}
RUN_FOREGROUND=false
DAEMON_AFFINITY_=${DAEMON_AFFINITY}
PROXY_AFFINITY_=${PROXY_AFFINITY}
USE_PROXY=${USE_PROXY}
# use absolute paths for everything
DAEMON_BIN=$(readlink -f ${DAEMON_BIN})
PROXY_BIN=$(readlink -f ${PROXY_BIN})
PRELOAD_LIB=$(readlink -f ${PRELOAD_LIB})
HOSTSFILE=$(readlink -f ${HOSTSFILE})
PROXY_LOCAL_PID_FILE=$(readlink -f ${PROXY_LOCAL_PID_FILE})
DAEMON_PID_FILE=$(readlink -f ${DAEMON_PID_FILE})
PROXY_PID_FILE=$(readlink -f ${PROXY_PID_FILE})
GKFS_MALLEABILITY_BIN_=${GKFS_MALLEABILITY_BIN}

# parse input
POSITIONAL=()
while [[ $# -gt 0 ]]; do
    key="$1"

    case ${key} in
    -r | --rootdir)
        ROOTDIR=$2
        shift # past argument
        shift # past value
        ;;
    -m | --mountdir)
        MOUNTDIR=$2
        shift # past argument
        shift # past value
        ;;
    -n | --numnodes)
        NODE_NUM=$2
        shift # past argument
        shift # past value
        ;;
    -d | --daemon_args)
        DAEMON_ARGS_="${DAEMON_ARGS_} $2"
        shift # past argument
        shift # past value
        ;;
    -p | --proxy_args)
        PROXY_ARGS_="${PROXY_ARGS_} $2"
        shift # past argument
        shift # past value
        ;;
    --proxy)
        USE_PROXY=true
        shift # past argument
        ;;
    --srun)
        USE_SRUN=true
        shift # past argument
        ;;
    -f | --foreground)
        RUN_FOREGROUND=true
        shift # past argument
        ;;
    --daemon_nodelist)
        DAEMON_NODELIST_="$2"
        shift # past argument
        shift # past value
        ;;
    --proxy_nodelist)
        PROXY_NODELIST_="$2"
        shift # past argument
        shift # past value
        ;;
    --cpuspertask)
        CPUS_PER_TASK=$2
        shift # past argument
        shift # past value
        ;;
    -c | --config)
            # skip. was handled above
            shift # past argument
            shift # past value
            ;;
    -H | --hostsfile)
            HOSTSFILE=$2
            shift # past argument
            shift # past value
            ;;
    -h | --help)
        help_msg
        exit
        ;;
    -v | --verbose)
        VERBOSE=true
        shift # past argument
        ;;
    -t | --time)
        SRUN_TIME=$2
        shift # past argument
        shift # past value
        ;;
    -A | --account)
        SLURM_ACCOUNT=$2
        shift # past argument
        shift # past value
        ;;
    -P | --partition)
        SLURM_PARTITION=$2
        shift # past argument
        shift # past value
        ;;
    *) # unknown option
        POSITIONAL+=("$1") # save it in an array for later
        shift              # past argument
        ;;
    esac
done
set -- "${POSITIONAL[@]}" # restore positional parameters
HOSTSFILE=$(readlink -f ${HOSTSFILE})

# positional arguments
if [[ -z ${1+x} ]]; then
    echo -e "${C_AST_RED}ERROR: Positional arguments missing."
    usage_short
    exit 1
fi
COMMAND="${1}"
# checking input
if [[ ${COMMAND} != "start" ]] && [[ ${COMMAND} != "stop" ]] && [[ ${COMMAND} != "mutate" ]] && [[ ${COMMAND} != "status" ]] && [[ ${COMMAND} != "finalize" ]]; then
    echo -e "${C_AST_RED}ERROR: COMMAND ${COMMAND} not supported"
    usage_short
    exit 1
fi
# Run script
if [[ ${COMMAND} == "start" ]]; then
    start_daemons
elif [[ ${COMMAND} == "stop" ]]; then
    stop_daemons
elif [[ ${COMMAND} == "mutate" ]]; then
    mutate_daemons
elif [[ ${COMMAND} == "status" ]]; then
    malleability_status
elif [[ ${COMMAND} == "finalize" ]]; then
    malleability_finalize
fi
if [[ ${VERBOSE} == true ]]; then
    echo -e "${C_AST_GREEN}Nothing left to do. Exiting :)"
fi
