Skip to content
Snippets Groups Projects
Commit 3dd6268c authored by Ramon Nou's avatar Ramon Nou
Browse files

xpn script

parent c26920b1
No related branches found
No related tags found
No related merge requests found
Pipeline #4547 canceled
Stage: build
    Stage: test
      Hide whitespace changes
      Inline Side-by-side
      plugins/adhoc_services.d/expand.sh
      + 339
      − 2
      View file @ 3dd6268c
      #!/usr/bin/env bash #!/bin/bash
      #set -x
      exit 0 #
      # Copyright 2020-2024 Felix Garcia Carballeira, Diego Camarmas Alonso, Alejandro Calderon Mateos, Elias del Pozo Puñal
      #
      # This file is part of Expand.
      #
      # Expand is free software: you can redistribute it and/or modify
      # it under the terms of the GNU Lesser General Public License as published by
      # the Free Software Foundation, either version 3 of the License, or
      # (at your option) any later version.
      #
      # Expand is distributed in the hope that it will be useful,
      # but WITHOUT ANY WARRANTY; without even the implied warranty of
      # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
      # GNU Lesser General Public License for more details.
      #
      # You should have received a copy of the GNU Lesser General Public License
      # along with Expand. If not, see <http://www.gnu.org/licenses/>.
      #
      usage_short() {
      echo ""
      echo " Usage: expand.sh [-h/--help] "
      echo " [-l/--hosts <host1,host2,...>] "
      echo " [-d/--datadir <path to datadir>]"
      echo " [-s/--sharedfir <path to sharedfir>]"
      echo " [-r/--replication_level <replication level>]"
      echo " [-v/--verbose <false>] {start,stop,expand,shrink}"
      echo ""
      }
      usage_details() {
      echo ""
      echo " This script simplifies the starting and stopping XPN ad-hoc servers."
      echo ""
      echo " positional arguments:"
      echo " command Command to execute: 'start', 'stop', 'expand' and 'shrink'"
      echo ""
      echo " optional arguments:"
      echo " -h, --help Shows this help message and exits."
      echo " -l, --hosts <arguments> A comma separated list of valid job hostnames that can be used to start the adhoc servers."
      echo " -d, --datadir <path> The default data directory for adhoc servers."
      echo " -s, --shareddir <path> The default shared directory for adhoc servers."
      echo " -r, --replication_level <path> The replication level to use."
      echo " -v, --verbose Increase verbosity"
      echo ""
      }
      get_opts() {
      # Taken the general idea from https://stackoverflow.com/questions/70951038/how-to-use-getopt-long-option-in-bash-script
      mkconf_name=$(basename "$0")
      mkconf_short_opt=l:d:s:r:vh
      mkconf_long_opt=hosts:,datadir:,shareddir:,replication_level:,verbose,help
      TEMP=$(getopt -o $mkconf_short_opt --long $mkconf_long_opt --name "$mkconf_name" -- "$@")
      eval set -- "${TEMP}"
      while :; do
      case "${1}" in
      -l | --hosts ) HOSTLIST=$2; shift 2 ;;
      -d | --datadir ) DATADIR=$2; shift 2 ;;
      -s | --shareddir ) SHAREDDIR=$2; shift 2 ;;
      -r | --replication_level) REPLICATION_LEVEL=$2; shift 2 ;;
      -v | --verbose ) VERBOSE=true; shift 1 ;;
      -h | --help ) usage_short; usage_details; exit 0 ;;
      -- ) shift; break ;;
      * ) intro; echo " > ERROR: parsing arguments found an error :-/"; usage; exit 1 ;;
      esac
      done
      ACTION=$*
      }
      ## default values
      ACTION=""
      HOSTLIST=$(hostname -I)
      DATADIR="/tmp/expand/data"
      SHAREDDIR=""
      DEPLOYMENTFILE=""
      NHOST=0
      REPLICATION_LEVEL=0
      VERBOSE=false
      ## get arguments
      BASE_DIR=$(dirname "$(readlink -f "$0")")/
      XPN_SH=${BASE_DIR}../../scripts/execute/xpn.sh
      get_opts $@
      if [ -z "$SHAREDDIR" ]; then
      echo "Error: shareddir not especified"
      usage_short
      exit 1
      fi
      HOSTFILE=${SHAREDDIR}/hostfile.txt
      HOSTFILE_REBUILD=${SHAREDDIR}/hostfile_rebuild.txt
      HOSTFILE_START=${SHAREDDIR}/hostfile_start.txt
      HOSTFILE_STOP=${SHAREDDIR}/hostfile_stop.txt
      HOSTFILE_REBUILD_JOIN=${SHAREDDIR}/hostfile_rebuild_join.txt
      if [[ ${VERBOSE} == true ]]; then
      echo "HOSTS=${HOSTLIST}"
      echo "DATADIR=${DATADIR}"
      echo "SHAREDDIR=${SHAREDDIR}"
      echo "REPLICATION_LEVEL=${REPLICATION_LEVEL}"
      start_script=$(date +%s%3N)
      fi
      # run
      case "${ACTION}" in
      start) rm -f ${HOSTFILE}
      touch ${HOSTFILE}
      rm -f ${SHAREDDIR}/dns.txt
      touch ${SHAREDDIR}/dns.txt
      # HOSTLIST -> HOSTFILE
      for i in $(echo ${HOSTLIST} | tr "," "\n")
      do
      NHOST=$((NHOST+1))
      echo $i >> ${HOSTFILE}
      done
      # xpn ...
      ${XPN_SH} --numnodes $NHOST --hostfile ${HOSTFILE} --xpn_storage_path ${DATADIR} --workdir ${SHAREDDIR} --replication_level ${REPLICATION_LEVEL} start
      ;;
      stop) ${XPN_SH} --deathfile ${HOSTFILE} --workdir ${SHAREDDIR} stop
      rm -f ${HOSTFILE}
      ;;
      expand_v2 | shrink_v2)
      rm -f ${HOSTFILE_REBUILD}
      touch ${HOSTFILE_REBUILD}
      # HOSTLIST -> HOSTFILE
      for i in $(echo ${HOSTLIST} | tr "," "\n")
      do
      NHOST=$((NHOST+1))
      echo $i >> ${HOSTFILE_REBUILD}
      done
      NHOST_OLD=$(cat ${HOSTFILE} | wc -l)
      diff ${HOSTFILE} ${HOSTFILE_REBUILD} | grep "^>" | cut -d' ' -f2 > ${HOSTFILE_START}
      diff ${HOSTFILE} ${HOSTFILE_REBUILD} | grep "^<" | cut -d' ' -f2 > ${HOSTFILE_STOP}
      NHOST_START=$(cat ${HOSTFILE_START} | wc -l)
      NHOST_STOP=$(cat ${HOSTFILE_STOP} | wc -l)
      if [ $NHOST_START -eq 0 ] && [ $NHOST_STOP -eq 0 ]; then
      echo "The hosts provided are the same"
      exit 0
      fi
      if [[ ${VERBOSE} == true ]]; then
      start_stop=$(date +%s%3N)
      fi
      if [ $NHOST_START -gt 0 ]; then
      ${XPN_SH} --numnodes ${NHOST_START} --hostfile ${HOSTFILE_START} --xpn_storage_path ${DATADIR} --workdir ${SHAREDDIR} --replication_level ${REPLICATION_LEVEL} start
      fi
      if [ $NHOST_STOP -gt 0 ]; then
      ${XPN_SH} --deathfile ${HOSTFILE_STOP} --workdir ${SHAREDDIR} stop
      fi
      if [[ ${VERBOSE} == true ]]; then
      end_stop=$(date +%s%3N)
      seconds=$(((end_stop - start_stop) / 1000))
      milliseconds=$(((end_stop - start_stop) % 1000))
      echo "Time to run start-stop: $seconds.$milliseconds sec"
      fi
      cat ${HOSTFILE} ${HOSTFILE_REBUILD} | sort | uniq > ${HOSTFILE_REBUILD_JOIN}
      NHOST_REBUILD_JOIN=$(cat ${HOSTFILE_REBUILD_JOIN} | wc -l)
      if [[ ${VERBOSE} == true ]]; then
      start_rebuild=$(date +%s%3N)
      fi
      ${XPN_SH} --numnodes ${NHOST_REBUILD_JOIN} --hostfile ${HOSTFILE_REBUILD_JOIN} --workdir ${SHAREDDIR} --xpn_storage_path ${DATADIR} --replication_level ${REPLICATION_LEVEL} --deathfile ${HOSTFILE} --rebuildfile ${HOSTFILE_REBUILD} rebuild
      if [[ ${VERBOSE} == true ]]; then
      end_rebuild=$(date +%s%3N)
      seconds=$(((end_rebuild - start_rebuild) / 1000))
      milliseconds=$(((end_rebuild - start_rebuild) % 1000))
      echo "Time to run rebuild: $seconds.$milliseconds sec"
      fi
      mv ${HOSTFILE_REBUILD} ${HOSTFILE}
      rm ${HOSTFILE_START}
      rm ${HOSTFILE_STOP}
      rm ${HOSTFILE_REBUILD_JOIN}
      ;;
      expand_v1 | shrink_v1)
      rm -f ${HOSTFILE_REBUILD}
      touch ${HOSTFILE_REBUILD}
      # HOSTLIST -> HOSTFILE
      for i in $(echo ${HOSTLIST} | tr "," "\n")
      do
      NHOST=$((NHOST+1))
      echo $i >> ${HOSTFILE_REBUILD}
      done
      NHOST_OLD=$(cat ${HOSTFILE} | wc -l)
      diff ${HOSTFILE} ${HOSTFILE_REBUILD} | grep "^>" | cut -d' ' -f2 > ${HOSTFILE_START}
      diff ${HOSTFILE} ${HOSTFILE_REBUILD} | grep "^<" | cut -d' ' -f2 > ${HOSTFILE_STOP}
      NHOST_START=$(cat ${HOSTFILE_START} | wc -l)
      NHOST_STOP=$(cat ${HOSTFILE_STOP} | wc -l)
      if [ $NHOST_START -eq 0 ] && [ $NHOST_STOP -eq 0 ]; then
      echo "The hosts provided are the same"
      exit 0
      fi
      # stop
      if [[ ${VERBOSE} == true ]]; then
      start_stop=$(date +%s%3N)
      fi
      ${XPN_SH} --deathfile ${HOSTFILE} --workdir ${SHAREDDIR} stop
      if [[ ${VERBOSE} == true ]]; then
      end_stop=$(date +%s%3N)
      seconds=$(((end_stop - start_stop) / 1000))
      milliseconds=$(((end_stop - start_stop) % 1000))
      echo "Time to run stop: $seconds.$milliseconds sec"
      fi
      cat ${HOSTFILE} ${HOSTFILE_REBUILD} | sort | uniq > ${HOSTFILE_REBUILD_JOIN}
      NHOST_REBUILD_JOIN=$(cat ${HOSTFILE_REBUILD_JOIN} | wc -l)
      if [[ ${VERBOSE} == true ]]; then
      start_rebuild=$(date +%s%3N)
      fi
      ${XPN_SH} --numnodes ${NHOST_REBUILD_JOIN} --hostfile ${HOSTFILE_REBUILD_JOIN} --workdir ${SHAREDDIR} --xpn_storage_path ${DATADIR} --replication_level ${REPLICATION_LEVEL} --deathfile ${HOSTFILE} --rebuildfile ${HOSTFILE_REBUILD} rebuild
      if [[ ${VERBOSE} == true ]]; then
      end_rebuild=$(date +%s%3N)
      seconds=$(((end_rebuild - start_rebuild) / 1000))
      milliseconds=$(((end_rebuild - start_rebuild) % 1000))
      echo "Time to run rebuild: $seconds.$milliseconds sec"
      fi
      # start
      if [[ ${VERBOSE} == true ]]; then
      start_start=$(date +%s%3N)
      fi
      mv ${HOSTFILE_REBUILD} ${HOSTFILE}
      ${XPN_SH} --numnodes $NHOST --hostfile ${HOSTFILE} --xpn_storage_path ${DATADIR} --workdir ${SHAREDDIR} --replication_level ${REPLICATION_LEVEL} start
      if [[ ${VERBOSE} == true ]]; then
      end_start=$(date +%s%3N)
      seconds=$(((end_start - start_start) / 1000))
      milliseconds=$(((end_start - start_start) % 1000))
      echo "Time to run start: $seconds.$milliseconds sec"
      fi
      rm ${HOSTFILE_START}
      rm ${HOSTFILE_STOP}
      rm ${HOSTFILE_REBUILD_JOIN}
      ;;
      expand | shrink)
      rm -f ${HOSTFILE_REBUILD}
      touch ${HOSTFILE_REBUILD}
      # HOSTLIST -> HOSTFILE
      for i in $(echo ${HOSTLIST} | tr "," "\n")
      do
      NHOST=$((NHOST+1))
      echo $i >> ${HOSTFILE_REBUILD}
      done
      NHOST_OLD=$(cat ${HOSTFILE} | wc -l)
      # stop
      if [[ ${VERBOSE} == true ]]; then
      start_stop=$(date +%s%3N)
      fi
      ${XPN_SH} --deathfile ${HOSTFILE} --workdir ${SHAREDDIR} stop
      if [[ ${VERBOSE} == true ]]; then
      end_stop=$(date +%s%3N)
      seconds=$(((end_stop - start_stop) / 1000))
      milliseconds=$(((end_stop - start_stop) % 1000))
      echo "Time to run stop: $seconds.$milliseconds sec"
      fi
      # flush
      if [[ ${VERBOSE} == true ]]; then
      start_flush=$(date +%s%3N)
      fi
      mkdir -p ${SHAREDDIR}/tmp_shrink
      ${XPN_SH} --numnodes $NHOST_OLD --hostfile ${HOSTFILE} --xpn_storage_path ${DATADIR} --destination_path ${SHAREDDIR}/tmp_shrink --replication_level ${REPLICATION_LEVEL} flush
      if [[ ${VERBOSE} == true ]]; then
      end_flush=$(date +%s%3N)
      seconds=$(((end_flush - start_flush) / 1000))
      milliseconds=$(((end_flush - start_flush) % 1000))
      echo "Time to run flush: $seconds.$milliseconds sec"
      fi
      # start
      if [[ ${VERBOSE} == true ]]; then
      start_start=$(date +%s%3N)
      fi
      mv ${HOSTFILE_REBUILD} ${HOSTFILE}
      ${XPN_SH} --numnodes $NHOST --hostfile ${HOSTFILE} --xpn_storage_path ${DATADIR} --workdir ${SHAREDDIR} --replication_level ${REPLICATION_LEVEL} start
      if [[ ${VERBOSE} == true ]]; then
      end_start=$(date +%s%3N)
      seconds=$(((end_start - start_start) / 1000))
      milliseconds=$(((end_start - start_start) % 1000))
      echo "Time to run start: $seconds.$milliseconds sec"
      fi
      # preload
      if [[ ${VERBOSE} == true ]]; then
      start_preload=$(date +%s%3N)
      fi
      ${XPN_SH} --numnodes $NHOST --hostfile ${HOSTFILE} --source_path ${SHAREDDIR}/tmp_shrink --xpn_storage_path ${DATADIR} --replication_level ${REPLICATION_LEVEL} preload
      rm -r ${SHAREDDIR}/tmp_shrink
      if [[ ${VERBOSE} == true ]]; then
      end_preload=$(date +%s%3N)
      seconds=$(((end_preload - start_preload) / 1000))
      milliseconds=$(((end_preload - start_preload) % 1000))
      echo "Time to run preload: $seconds.$milliseconds sec"
      fi
      ;;
      *) echo ""
      echo " ERROR: ACTION '${ACTION}' not supported"
      usage_short
      exit 1
      ;;
      esac
      if [[ ${VERBOSE} == true ]]; then
      end_script=$(date +%s%3N)
      seconds=$(((end_script - start_script) / 1000))
      milliseconds=$(((end_script - start_script) % 1000))
      echo "Time to run script: $seconds.$milliseconds sec"
      fi
      \ No newline at end of file
      0% or .
      You are about to add 0 people to the discussion. Proceed with caution.
      Finish editing this message first!
      Please register or to comment