Skip to content
Snippets Groups Projects
Commit b5652a3d authored by Ramon Nou's avatar Ramon Nou
Browse files

xpn script

parent a50a7e6c
No related branches found
No related tags found
1 merge request!127165 xpn integration
Pipeline #4553 passed
Hide whitespace changes
Inline Side-by-side
plugins/adhoc_services.d/expand.sh
+ 339
− 2
View file @ b5652a3d
#!/usr/bin/env bash
#!/bin/bash
#set -x
exit 0
#
# Copyright 2020-2024 Felix Garcia Carballeira, Diego Camarmas Alonso, Alejandro Calderon Mateos, Elias del Pozo Puñal
#
# This file is part of Expand.
#
# Expand is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Expand is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Expand. If not, see <http://www.gnu.org/licenses/>.
#
usage_short() {
echo ""
echo " Usage: expand.sh [-h/--help] "
echo " [-l/--hosts <host1,host2,...>] "
echo " [-d/--datadir <path to datadir>]"
echo " [-s/--sharedfir <path to sharedfir>]"
echo " [-r/--replication_level <replication level>]"
echo " [-v/--verbose <false>] {start,stop,expand,shrink}"
echo ""
}
usage_details() {
echo ""
echo " This script simplifies the starting and stopping XPN ad-hoc servers."
echo ""
echo " positional arguments:"
echo " command Command to execute: 'start', 'stop', 'expand' and 'shrink'"
echo ""
echo " optional arguments:"
echo " -h, --help Shows this help message and exits."
echo " -l, --hosts <arguments> A comma separated list of valid job hostnames that can be used to start the adhoc servers."
echo " -d, --datadir <path> The default data directory for adhoc servers."
echo " -s, --shareddir <path> The default shared directory for adhoc servers."
echo " -r, --replication_level <path> The replication level to use."
echo " -v, --verbose Increase verbosity"
echo ""
}
get_opts() {
# Taken the general idea from https://stackoverflow.com/questions/70951038/how-to-use-getopt-long-option-in-bash-script
mkconf_name=$(basename "$0")
mkconf_short_opt=l:d:s:r:vh
mkconf_long_opt=hosts:,datadir:,shareddir:,replication_level:,verbose,help
TEMP=$(getopt -o $mkconf_short_opt --long $mkconf_long_opt --name "$mkconf_name" -- "$@")
eval set -- "${TEMP}"
while :; do
case "${1}" in
-l | --hosts ) HOSTLIST=$2; shift 2 ;;
-d | --datadir ) DATADIR=$2; shift 2 ;;
-s | --shareddir ) SHAREDDIR=$2; shift 2 ;;
-r | --replication_level) REPLICATION_LEVEL=$2; shift 2 ;;
-v | --verbose ) VERBOSE=true; shift 1 ;;
-h | --help ) usage_short; usage_details; exit 0 ;;
-- ) shift; break ;;
* ) intro; echo " > ERROR: parsing arguments found an error :-/"; usage; exit 1 ;;
esac
done
ACTION=$*
}
## default values
ACTION=""
HOSTLIST=$(hostname -I)
DATADIR="/tmp/expand/data"
SHAREDDIR=""
DEPLOYMENTFILE=""
NHOST=0
REPLICATION_LEVEL=0
VERBOSE=false
## get arguments
BASE_DIR=$(dirname "$(readlink -f "$0")")/
XPN_SH=${BASE_DIR}../../scripts/execute/xpn.sh
get_opts $@
if [ -z "$SHAREDDIR" ]; then
echo "Error: shareddir not especified"
usage_short
exit 1
fi
HOSTFILE=${SHAREDDIR}/hostfile.txt
HOSTFILE_REBUILD=${SHAREDDIR}/hostfile_rebuild.txt
HOSTFILE_START=${SHAREDDIR}/hostfile_start.txt
HOSTFILE_STOP=${SHAREDDIR}/hostfile_stop.txt
HOSTFILE_REBUILD_JOIN=${SHAREDDIR}/hostfile_rebuild_join.txt
if [[ ${VERBOSE} == true ]]; then
echo "HOSTS=${HOSTLIST}"
echo "DATADIR=${DATADIR}"
echo "SHAREDDIR=${SHAREDDIR}"
echo "REPLICATION_LEVEL=${REPLICATION_LEVEL}"
start_script=$(date +%s%3N)
fi
# run
case "${ACTION}" in
start) rm -f ${HOSTFILE}
touch ${HOSTFILE}
rm -f ${SHAREDDIR}/dns.txt
touch ${SHAREDDIR}/dns.txt
# HOSTLIST -> HOSTFILE
for i in $(echo ${HOSTLIST} | tr "," "\n")
do
NHOST=$((NHOST+1))
echo $i >> ${HOSTFILE}
done
# xpn ...
${XPN_SH} --numnodes $NHOST --hostfile ${HOSTFILE} --xpn_storage_path ${DATADIR} --workdir ${SHAREDDIR} --replication_level ${REPLICATION_LEVEL} start
;;
stop) ${XPN_SH} --deathfile ${HOSTFILE} --workdir ${SHAREDDIR} stop
rm -f ${HOSTFILE}
;;
expand_v2 | shrink_v2)
rm -f ${HOSTFILE_REBUILD}
touch ${HOSTFILE_REBUILD}
# HOSTLIST -> HOSTFILE
for i in $(echo ${HOSTLIST} | tr "," "\n")
do
NHOST=$((NHOST+1))
echo $i >> ${HOSTFILE_REBUILD}
done
NHOST_OLD=$(cat ${HOSTFILE} | wc -l)
diff ${HOSTFILE} ${HOSTFILE_REBUILD} | grep "^>" | cut -d' ' -f2 > ${HOSTFILE_START}
diff ${HOSTFILE} ${HOSTFILE_REBUILD} | grep "^<" | cut -d' ' -f2 > ${HOSTFILE_STOP}
NHOST_START=$(cat ${HOSTFILE_START} | wc -l)
NHOST_STOP=$(cat ${HOSTFILE_STOP} | wc -l)
if [ $NHOST_START -eq 0 ] && [ $NHOST_STOP -eq 0 ]; then
echo "The hosts provided are the same"
exit 0
fi
if [[ ${VERBOSE} == true ]]; then
start_stop=$(date +%s%3N)
fi
if [ $NHOST_START -gt 0 ]; then
${XPN_SH} --numnodes ${NHOST_START} --hostfile ${HOSTFILE_START} --xpn_storage_path ${DATADIR} --workdir ${SHAREDDIR} --replication_level ${REPLICATION_LEVEL} start
fi
if [ $NHOST_STOP -gt 0 ]; then
${XPN_SH} --deathfile ${HOSTFILE_STOP} --workdir ${SHAREDDIR} stop
fi
if [[ ${VERBOSE} == true ]]; then
end_stop=$(date +%s%3N)
seconds=$(((end_stop - start_stop) / 1000))
milliseconds=$(((end_stop - start_stop) % 1000))
echo "Time to run start-stop: $seconds.$milliseconds sec"
fi
cat ${HOSTFILE} ${HOSTFILE_REBUILD} | sort | uniq > ${HOSTFILE_REBUILD_JOIN}
NHOST_REBUILD_JOIN=$(cat ${HOSTFILE_REBUILD_JOIN} | wc -l)
if [[ ${VERBOSE} == true ]]; then
start_rebuild=$(date +%s%3N)
fi
${XPN_SH} --numnodes ${NHOST_REBUILD_JOIN} --hostfile ${HOSTFILE_REBUILD_JOIN} --workdir ${SHAREDDIR} --xpn_storage_path ${DATADIR} --replication_level ${REPLICATION_LEVEL} --deathfile ${HOSTFILE} --rebuildfile ${HOSTFILE_REBUILD} rebuild
if [[ ${VERBOSE} == true ]]; then
end_rebuild=$(date +%s%3N)
seconds=$(((end_rebuild - start_rebuild) / 1000))
milliseconds=$(((end_rebuild - start_rebuild) % 1000))
echo "Time to run rebuild: $seconds.$milliseconds sec"
fi
mv ${HOSTFILE_REBUILD} ${HOSTFILE}
rm ${HOSTFILE_START}
rm ${HOSTFILE_STOP}
rm ${HOSTFILE_REBUILD_JOIN}
;;
expand_v1 | shrink_v1)
rm -f ${HOSTFILE_REBUILD}
touch ${HOSTFILE_REBUILD}
# HOSTLIST -> HOSTFILE
for i in $(echo ${HOSTLIST} | tr "," "\n")
do
NHOST=$((NHOST+1))
echo $i >> ${HOSTFILE_REBUILD}
done
NHOST_OLD=$(cat ${HOSTFILE} | wc -l)
diff ${HOSTFILE} ${HOSTFILE_REBUILD} | grep "^>" | cut -d' ' -f2 > ${HOSTFILE_START}
diff ${HOSTFILE} ${HOSTFILE_REBUILD} | grep "^<" | cut -d' ' -f2 > ${HOSTFILE_STOP}
NHOST_START=$(cat ${HOSTFILE_START} | wc -l)
NHOST_STOP=$(cat ${HOSTFILE_STOP} | wc -l)
if [ $NHOST_START -eq 0 ] && [ $NHOST_STOP -eq 0 ]; then
echo "The hosts provided are the same"
exit 0
fi
# stop
if [[ ${VERBOSE} == true ]]; then
start_stop=$(date +%s%3N)
fi
${XPN_SH} --deathfile ${HOSTFILE} --workdir ${SHAREDDIR} stop
if [[ ${VERBOSE} == true ]]; then
end_stop=$(date +%s%3N)
seconds=$(((end_stop - start_stop) / 1000))
milliseconds=$(((end_stop - start_stop) % 1000))
echo "Time to run stop: $seconds.$milliseconds sec"
fi
cat ${HOSTFILE} ${HOSTFILE_REBUILD} | sort | uniq > ${HOSTFILE_REBUILD_JOIN}
NHOST_REBUILD_JOIN=$(cat ${HOSTFILE_REBUILD_JOIN} | wc -l)
if [[ ${VERBOSE} == true ]]; then
start_rebuild=$(date +%s%3N)
fi
${XPN_SH} --numnodes ${NHOST_REBUILD_JOIN} --hostfile ${HOSTFILE_REBUILD_JOIN} --workdir ${SHAREDDIR} --xpn_storage_path ${DATADIR} --replication_level ${REPLICATION_LEVEL} --deathfile ${HOSTFILE} --rebuildfile ${HOSTFILE_REBUILD} rebuild
if [[ ${VERBOSE} == true ]]; then
end_rebuild=$(date +%s%3N)
seconds=$(((end_rebuild - start_rebuild) / 1000))
milliseconds=$(((end_rebuild - start_rebuild) % 1000))
echo "Time to run rebuild: $seconds.$milliseconds sec"
fi
# start
if [[ ${VERBOSE} == true ]]; then
start_start=$(date +%s%3N)
fi
mv ${HOSTFILE_REBUILD} ${HOSTFILE}
${XPN_SH} --numnodes $NHOST --hostfile ${HOSTFILE} --xpn_storage_path ${DATADIR} --workdir ${SHAREDDIR} --replication_level ${REPLICATION_LEVEL} start
if [[ ${VERBOSE} == true ]]; then
end_start=$(date +%s%3N)
seconds=$(((end_start - start_start) / 1000))
milliseconds=$(((end_start - start_start) % 1000))
echo "Time to run start: $seconds.$milliseconds sec"
fi
rm ${HOSTFILE_START}
rm ${HOSTFILE_STOP}
rm ${HOSTFILE_REBUILD_JOIN}
;;
expand | shrink)
rm -f ${HOSTFILE_REBUILD}
touch ${HOSTFILE_REBUILD}
# HOSTLIST -> HOSTFILE
for i in $(echo ${HOSTLIST} | tr "," "\n")
do
NHOST=$((NHOST+1))
echo $i >> ${HOSTFILE_REBUILD}
done
NHOST_OLD=$(cat ${HOSTFILE} | wc -l)
# stop
if [[ ${VERBOSE} == true ]]; then
start_stop=$(date +%s%3N)
fi
${XPN_SH} --deathfile ${HOSTFILE} --workdir ${SHAREDDIR} stop
if [[ ${VERBOSE} == true ]]; then
end_stop=$(date +%s%3N)
seconds=$(((end_stop - start_stop) / 1000))
milliseconds=$(((end_stop - start_stop) % 1000))
echo "Time to run stop: $seconds.$milliseconds sec"
fi
# flush
if [[ ${VERBOSE} == true ]]; then
start_flush=$(date +%s%3N)
fi
mkdir -p ${SHAREDDIR}/tmp_shrink
${XPN_SH} --numnodes $NHOST_OLD --hostfile ${HOSTFILE} --xpn_storage_path ${DATADIR} --destination_path ${SHAREDDIR}/tmp_shrink --replication_level ${REPLICATION_LEVEL} flush
if [[ ${VERBOSE} == true ]]; then
end_flush=$(date +%s%3N)
seconds=$(((end_flush - start_flush) / 1000))
milliseconds=$(((end_flush - start_flush) % 1000))
echo "Time to run flush: $seconds.$milliseconds sec"
fi
# start
if [[ ${VERBOSE} == true ]]; then
start_start=$(date +%s%3N)
fi
mv ${HOSTFILE_REBUILD} ${HOSTFILE}
${XPN_SH} --numnodes $NHOST --hostfile ${HOSTFILE} --xpn_storage_path ${DATADIR} --workdir ${SHAREDDIR} --replication_level ${REPLICATION_LEVEL} start
if [[ ${VERBOSE} == true ]]; then
end_start=$(date +%s%3N)
seconds=$(((end_start - start_start) / 1000))
milliseconds=$(((end_start - start_start) % 1000))
echo "Time to run start: $seconds.$milliseconds sec"
fi
# preload
if [[ ${VERBOSE} == true ]]; then
start_preload=$(date +%s%3N)
fi
${XPN_SH} --numnodes $NHOST --hostfile ${HOSTFILE} --source_path ${SHAREDDIR}/tmp_shrink --xpn_storage_path ${DATADIR} --replication_level ${REPLICATION_LEVEL} preload
rm -r ${SHAREDDIR}/tmp_shrink
if [[ ${VERBOSE} == true ]]; then
end_preload=$(date +%s%3N)
seconds=$(((end_preload - start_preload) / 1000))
milliseconds=$(((end_preload - start_preload) % 1000))
echo "Time to run preload: $seconds.$milliseconds sec"
fi
;;
*) echo ""
echo " ERROR: ACTION '${ACTION}' not supported"
usage_short
exit 1
;;
esac
if [[ ${VERBOSE} == true ]]; then
end_script=$(date +%s%3N)
seconds=$(((end_script - start_script) / 1000))
milliseconds=$(((end_script - start_script) % 1000))
echo "Time to run script: $seconds.$milliseconds sec"
fi
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment