Merge branch 'rnou/165-xpn-integration' into 'main'

165 xpn integration

See merge request !127

plugins/adhoc_services.d/expand.sh

-#!/usr/bin/env bash
+#!/bin/bash
+#set -x
 
-exit 0
+#
+#  Copyright 2020-2024 Felix Garcia Carballeira, Diego Camarmas Alonso, Alejandro Calderon Mateos, Elias del Pozo Puñal
+#
+#  This file is part of Expand.
+#
+#  Expand is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Lesser General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  Expand is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public License
+#  along with Expand.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+usage_short() {
+  echo ""
+  echo " Usage: expand.sh [-h/--help] "
+  echo "                  [-l/--hosts    <host1,host2,...>] "
+  echo "                  [-d/--datadir  <path to datadir>]"
+  echo "                  [-s/--sharedfir <path to sharedfir>]"
+  echo "                  [-r/--replication_level <replication level>]"
+  echo "                  [-v/--verbose <false>] {start,stop,expand,shrink}"
+  echo ""
+}
+
+
+usage_details() {
+  echo ""
+  echo " This script simplifies the starting and stopping XPN ad-hoc servers."
+  echo ""
+  echo " positional arguments:"
+  echo "     command                  Command to execute: 'start', 'stop', 'expand' and 'shrink'"
+  echo ""
+  echo " optional arguments:"
+  echo "     -h, --help                               Shows this help message and exits."
+  echo "     -l, --hosts              <arguments>     A comma separated list of valid job hostnames that can be used to start the adhoc servers."
+  echo "     -d, --datadir            <path>          The default data directory for adhoc servers."
+  echo "     -s, --shareddir          <path>          The default shared directory for adhoc servers."
+  echo "     -r, --replication_level  <path>          The replication level to use."
+  echo "     -v, --verbose                            Increase verbosity"
+  echo ""
+}
+
+get_opts() {
+   # Taken the general idea from https://stackoverflow.com/questions/70951038/how-to-use-getopt-long-option-in-bash-script
+   mkconf_name=$(basename "$0")
+   mkconf_short_opt=l:d:s:r:vh
+   mkconf_long_opt=hosts:,datadir:,shareddir:,replication_level:,verbose,help
+   TEMP=$(getopt -o $mkconf_short_opt --long $mkconf_long_opt --name "$mkconf_name" -- "$@")
+   eval set -- "${TEMP}"
+
+   while :; do
+      case "${1}" in
+         -l | --hosts            ) HOSTLIST=$2;                 shift 2 ;;
+         -d | --datadir          ) DATADIR=$2;                  shift 2 ;;
+         -s | --shareddir        ) SHAREDDIR=$2;                shift 2 ;;
+         -r | --replication_level) REPLICATION_LEVEL=$2;        shift 2 ;;
+         -v | --verbose          ) VERBOSE=true;                shift 1 ;;
+         -h | --help             ) usage_short; usage_details;  exit 0 ;;
+         --                      ) shift;         break  ;;
+         *                       ) intro; echo " > ERROR: parsing arguments found an error :-/"; usage; exit 1 ;;
+      esac
+   done
+
+   ACTION=$*
+}
+
+## default values
+ACTION=""
+HOSTLIST=$(hostname -I)
+DATADIR="/tmp/expand/data"
+SHAREDDIR=""
+DEPLOYMENTFILE=""
+NHOST=0
+REPLICATION_LEVEL=0
+VERBOSE=false
+
+## get arguments
+BASE_DIR=$(dirname "$(readlink -f "$0")")/
+XPN_SH=${BASE_DIR}../../scripts/execute/xpn.sh
+get_opts $@
+
+if [ -z "$SHAREDDIR" ]; then
+  echo "Error: shareddir not especified"
+  usage_short
+  exit 1
+fi
+
+HOSTFILE=${SHAREDDIR}/hostfile.txt
+HOSTFILE_REBUILD=${SHAREDDIR}/hostfile_rebuild.txt
+HOSTFILE_START=${SHAREDDIR}/hostfile_start.txt
+HOSTFILE_STOP=${SHAREDDIR}/hostfile_stop.txt
+HOSTFILE_REBUILD_JOIN=${SHAREDDIR}/hostfile_rebuild_join.txt
+
+if [[ ${VERBOSE} == true ]]; then
+  echo "HOSTS=${HOSTLIST}"
+  echo "DATADIR=${DATADIR}"
+  echo "SHAREDDIR=${SHAREDDIR}"
+  echo "REPLICATION_LEVEL=${REPLICATION_LEVEL}"
+  start_script=$(date +%s%3N)
+fi
+# run 
+case "${ACTION}" in
+      start)    rm -f  ${HOSTFILE}
+                touch  ${HOSTFILE}
+                rm -f  ${SHAREDDIR}/dns.txt
+                touch  ${SHAREDDIR}/dns.txt
+                # HOSTLIST -> HOSTFILE
+                for i in $(echo ${HOSTLIST} | tr "," "\n")
+                do
+                  NHOST=$((NHOST+1))
+                  echo $i >> ${HOSTFILE}
+                done
+                
+                # xpn ...
+                ${XPN_SH} --numnodes $NHOST --hostfile ${HOSTFILE} --xpn_storage_path ${DATADIR} --workdir ${SHAREDDIR} --replication_level ${REPLICATION_LEVEL} start
+                ;;
+
+      stop)     ${XPN_SH} --deathfile ${HOSTFILE} --workdir ${SHAREDDIR} stop
+                rm -f ${HOSTFILE}
+                ;;
+
+      expand_v2 | shrink_v2)   
+                rm -f  ${HOSTFILE_REBUILD}
+                touch  ${HOSTFILE_REBUILD}
+                # HOSTLIST -> HOSTFILE
+                for i in $(echo ${HOSTLIST} | tr "," "\n")
+                do
+                  NHOST=$((NHOST+1))
+                  echo $i >> ${HOSTFILE_REBUILD}
+                done
+                NHOST_OLD=$(cat ${HOSTFILE} | wc -l)
+
+                diff ${HOSTFILE} ${HOSTFILE_REBUILD} | grep "^>" | cut -d' ' -f2 > ${HOSTFILE_START}
+                diff ${HOSTFILE} ${HOSTFILE_REBUILD} | grep "^<" | cut -d' ' -f2 > ${HOSTFILE_STOP}
+                NHOST_START=$(cat ${HOSTFILE_START} | wc -l)
+                NHOST_STOP=$(cat ${HOSTFILE_STOP} | wc -l)
+
+                if [ $NHOST_START -eq 0 ] && [ $NHOST_STOP -eq 0 ]; then
+                    echo "The hosts provided are the same"
+                    exit 0
+                fi
+
+                if [[ ${VERBOSE} == true ]]; then
+                  start_stop=$(date +%s%3N)
+                fi
+
+                if [ $NHOST_START -gt 0 ]; then
+                  ${XPN_SH} --numnodes ${NHOST_START} --hostfile ${HOSTFILE_START} --xpn_storage_path ${DATADIR} --workdir ${SHAREDDIR} --replication_level ${REPLICATION_LEVEL} start
+                fi
+                
+                if [ $NHOST_STOP -gt 0 ]; then
+                  ${XPN_SH} --deathfile ${HOSTFILE_STOP} --workdir ${SHAREDDIR} stop
+                fi
+
+                if [[ ${VERBOSE} == true ]]; then
+                  end_stop=$(date +%s%3N)
+                  seconds=$(((end_stop - start_stop) / 1000))
+                  milliseconds=$(((end_stop - start_stop) % 1000))
+                  echo "Time to run start-stop: $seconds.$milliseconds sec"
+                fi
+                
+                cat ${HOSTFILE} ${HOSTFILE_REBUILD} | sort | uniq > ${HOSTFILE_REBUILD_JOIN}
+                NHOST_REBUILD_JOIN=$(cat ${HOSTFILE_REBUILD_JOIN} | wc -l)
+
+                if [[ ${VERBOSE} == true ]]; then
+                  start_rebuild=$(date +%s%3N)
+                fi
+
+                ${XPN_SH} --numnodes ${NHOST_REBUILD_JOIN} --hostfile ${HOSTFILE_REBUILD_JOIN} --workdir ${SHAREDDIR} --xpn_storage_path ${DATADIR} --replication_level ${REPLICATION_LEVEL} --deathfile ${HOSTFILE} --rebuildfile ${HOSTFILE_REBUILD} rebuild
+
+                if [[ ${VERBOSE} == true ]]; then
+                  end_rebuild=$(date +%s%3N)
+                  seconds=$(((end_rebuild - start_rebuild) / 1000))
+                  milliseconds=$(((end_rebuild - start_rebuild) % 1000))
+                  echo "Time to run rebuild: $seconds.$milliseconds sec"
+                fi
+
+                mv ${HOSTFILE_REBUILD} ${HOSTFILE}
+                rm ${HOSTFILE_START}
+                rm ${HOSTFILE_STOP}
+                rm ${HOSTFILE_REBUILD_JOIN}
+                ;;
+      expand_v1 | shrink_v1)   
+                rm -f  ${HOSTFILE_REBUILD}
+                touch  ${HOSTFILE_REBUILD}
+                # HOSTLIST -> HOSTFILE
+                for i in $(echo ${HOSTLIST} | tr "," "\n")
+                do
+                  NHOST=$((NHOST+1))
+                  echo $i >> ${HOSTFILE_REBUILD}
+                done
+                NHOST_OLD=$(cat ${HOSTFILE} | wc -l)
+
+                diff ${HOSTFILE} ${HOSTFILE_REBUILD} | grep "^>" | cut -d' ' -f2 > ${HOSTFILE_START}
+                diff ${HOSTFILE} ${HOSTFILE_REBUILD} | grep "^<" | cut -d' ' -f2 > ${HOSTFILE_STOP}
+                NHOST_START=$(cat ${HOSTFILE_START} | wc -l)
+                NHOST_STOP=$(cat ${HOSTFILE_STOP} | wc -l)
+
+                if [ $NHOST_START -eq 0 ] && [ $NHOST_STOP -eq 0 ]; then
+                    echo "The hosts provided are the same"
+                    exit 0
+                fi
+                # stop
+                if [[ ${VERBOSE} == true ]]; then
+                  start_stop=$(date +%s%3N)
+                fi
+
+                ${XPN_SH} --deathfile ${HOSTFILE} --workdir ${SHAREDDIR} stop
+                
+                if [[ ${VERBOSE} == true ]]; then
+                  end_stop=$(date +%s%3N)
+                  seconds=$(((end_stop - start_stop) / 1000))
+                  milliseconds=$(((end_stop - start_stop) % 1000))
+                  echo "Time to run stop: $seconds.$milliseconds sec"
+                fi
+                
+                cat ${HOSTFILE} ${HOSTFILE_REBUILD} | sort | uniq > ${HOSTFILE_REBUILD_JOIN}
+                NHOST_REBUILD_JOIN=$(cat ${HOSTFILE_REBUILD_JOIN} | wc -l)
+
+                if [[ ${VERBOSE} == true ]]; then
+                  start_rebuild=$(date +%s%3N)
+                fi
+
+                ${XPN_SH} --numnodes ${NHOST_REBUILD_JOIN} --hostfile ${HOSTFILE_REBUILD_JOIN} --workdir ${SHAREDDIR} --xpn_storage_path ${DATADIR} --replication_level ${REPLICATION_LEVEL} --deathfile ${HOSTFILE} --rebuildfile ${HOSTFILE_REBUILD} rebuild
+
+                if [[ ${VERBOSE} == true ]]; then
+                  end_rebuild=$(date +%s%3N)
+                  seconds=$(((end_rebuild - start_rebuild) / 1000))
+                  milliseconds=$(((end_rebuild - start_rebuild) % 1000))
+                  echo "Time to run rebuild: $seconds.$milliseconds sec"
+                fi
+
+                # start
+                if [[ ${VERBOSE} == true ]]; then
+                  start_start=$(date +%s%3N)
+                fi
+
+                mv ${HOSTFILE_REBUILD} ${HOSTFILE}
+                ${XPN_SH} --numnodes $NHOST --hostfile ${HOSTFILE} --xpn_storage_path ${DATADIR} --workdir ${SHAREDDIR} --replication_level ${REPLICATION_LEVEL} start
+                
+                if [[ ${VERBOSE} == true ]]; then
+                  end_start=$(date +%s%3N)
+                  seconds=$(((end_start - start_start) / 1000))
+                  milliseconds=$(((end_start - start_start) % 1000))
+                  echo "Time to run start: $seconds.$milliseconds sec"
+                fi
+
+                rm ${HOSTFILE_START}
+                rm ${HOSTFILE_STOP}
+                rm ${HOSTFILE_REBUILD_JOIN}
+                ;;
+      expand | shrink)   
+                rm -f  ${HOSTFILE_REBUILD}
+                touch  ${HOSTFILE_REBUILD}
+                # HOSTLIST -> HOSTFILE
+                for i in $(echo ${HOSTLIST} | tr "," "\n")
+                do
+                  NHOST=$((NHOST+1))
+                  echo $i >> ${HOSTFILE_REBUILD}
+                done
+                NHOST_OLD=$(cat ${HOSTFILE} | wc -l)
+                # stop
+                if [[ ${VERBOSE} == true ]]; then
+                  start_stop=$(date +%s%3N)
+                fi
+
+                ${XPN_SH} --deathfile ${HOSTFILE} --workdir ${SHAREDDIR} stop
+                
+                if [[ ${VERBOSE} == true ]]; then
+                  end_stop=$(date +%s%3N)
+                  seconds=$(((end_stop - start_stop) / 1000))
+                  milliseconds=$(((end_stop - start_stop) % 1000))
+                  echo "Time to run stop: $seconds.$milliseconds sec"
+                fi
+                # flush
+                if [[ ${VERBOSE} == true ]]; then
+                  start_flush=$(date +%s%3N)
+                fi
+
+                mkdir -p ${SHAREDDIR}/tmp_shrink
+                ${XPN_SH} --numnodes $NHOST_OLD --hostfile ${HOSTFILE} --xpn_storage_path ${DATADIR} --destination_path ${SHAREDDIR}/tmp_shrink --replication_level ${REPLICATION_LEVEL} flush
+                
+                if [[ ${VERBOSE} == true ]]; then
+                  end_flush=$(date +%s%3N)
+                  seconds=$(((end_flush - start_flush) / 1000))
+                  milliseconds=$(((end_flush - start_flush) % 1000))
+                  echo "Time to run flush: $seconds.$milliseconds sec"
+                fi
+                # start
+                if [[ ${VERBOSE} == true ]]; then
+                  start_start=$(date +%s%3N)
+                fi
+
+                mv ${HOSTFILE_REBUILD} ${HOSTFILE}
+                ${XPN_SH} --numnodes $NHOST --hostfile ${HOSTFILE} --xpn_storage_path ${DATADIR} --workdir ${SHAREDDIR} --replication_level ${REPLICATION_LEVEL} start
+                
+                if [[ ${VERBOSE} == true ]]; then
+                  end_start=$(date +%s%3N)
+                  seconds=$(((end_start - start_start) / 1000))
+                  milliseconds=$(((end_start - start_start) % 1000))
+                  echo "Time to run start: $seconds.$milliseconds sec"
+                fi
+                # preload
+                if [[ ${VERBOSE} == true ]]; then
+                  start_preload=$(date +%s%3N)
+                fi
+
+                ${XPN_SH} --numnodes $NHOST --hostfile ${HOSTFILE} --source_path ${SHAREDDIR}/tmp_shrink --xpn_storage_path ${DATADIR} --replication_level ${REPLICATION_LEVEL} preload
+                rm -r ${SHAREDDIR}/tmp_shrink
+                
+                if [[ ${VERBOSE} == true ]]; then
+                  end_preload=$(date +%s%3N)
+                  seconds=$(((end_preload - start_preload) / 1000))
+                  milliseconds=$(((end_preload - start_preload) % 1000))
+                  echo "Time to run preload: $seconds.$milliseconds sec"
+                fi
+                ;;
+      *)        echo ""
+                echo " ERROR: ACTION '${ACTION}' not supported"
+                usage_short
+                exit 1
+                ;;
+esac
+
+if [[ ${VERBOSE} == true ]]; then
+  end_script=$(date +%s%3N)
+  seconds=$(((end_script - start_script) / 1000))
+  milliseconds=$(((end_script - start_script) % 1000))
+  echo "Time to run script: $seconds.$milliseconds sec"
+fi
\ No newline at end of file