Verified Commit d6724f78 authored by Marc Vef's avatar Marc Vef
Browse files

Finalizing gkfs script and configs

parent 59065b2e

scripts/run/gkfs

+18 −0
Original line number Diff line number Diff line
@@ -141,9 +141,18 @@ start_daemon() { 
        fi

    else

        NODE_NUM=1

        if [[ ${USE_PROXY} == true ]]; then

            NODE_NUM_PROXY=$NODE_NUM

        fi

    fi



    if [[ ${VERBOSE} == true ]]; then

        echo -e "${C_AST_GREEN}-------------------CLIENT SETTINGS------------------------"

        echo -e "${C_AST_GREEN}Set env variables for client: "

        echo -e "LD_PRELOAD=${PRELOAD_LIB}"

        echo -e "LIBGKFS_HOSTS_FILE=${HOSTSFILE}"

        echo -e "LIBGKFS_PROXY_PID_FILE=${PROXY_LOCAL_PID_FILE}"

        echo -e "${C_AST_GREEN}----------------------------------------------------------"

        echo -e "${C_AST_GREEN}mountdir: ${MOUNTDIR}"

        echo -e "${C_AST_GREEN}rootdir: ${ROOTDIR}"

        echo -e "${C_AST_GREEN}node_num: ${NODE_NUM}"
@@ -393,6 +402,15 @@ RUN_FOREGROUND=false 
DAEMON_AFFINITY_=${DAEMON_AFFINITY}

PROXY_AFFINITY_=${PROXY_AFFINITY}

USE_PROXY=${USE_PROXY}

# use absolute paths for everything

DAEMON_BIN=$(readlink -f ${DAEMON_BIN})

PROXY_BIN=$(readlink -f ${PROXY_BIN})

PRELOAD_LIB=$(readlink -f ${PRELOAD_LIB})

HOSTSFILE=$(readlink -f ${HOSTSFILE})

PROXY_LOCAL_PID_FILE=$(readlink -f ${PROXY_LOCAL_PID_FILE})

SRUN_DAEMON_PID_FILE=$(readlink -f ${SRUN_DAEMON_PID_FILE})

SRUN_PROXY_PID_FILE=$(readlink -f ${SRUN_PROXY_PID_FILE})



# parse input

POSITIONAL=()

while [[ $# -gt 0 ]]; do

scripts/run/gkfs.conf

+33 −16
Original line number Diff line number Diff line
@@ -3,30 +3,47 @@ 
# binaries (default for project_dir/build

PRELOAD_LIB=../../build/src/client/libgkfs_intercept.so

DAEMON_BIN=../../build/src/daemon/gkfs_daemon

PROXY_BIN=../../build/src/proxy/gkfs_proxy



# client configuration

# client configuration (needs to be set for all clients)

LIBGKFS_HOSTS_FILE=./gkfs_hostfile



# daemon configuration

DAEMON_ROOTDIR=/dev/shm/gkfs_rootdir

DAEMON_MOUNTDIR=/dev/shm/gkfs_mountdir

# path to daemon pid file; created where the script is run

DAEMON_PID_FILE=./gkfs_daemon.pid

## daemon configuration

DAEMON_ROOTDIR=/dev/shm/vef_gkfs_rootdir

DAEMON_MOUNTDIR=/dev/shm/vef_gkfs_mountdir

# additional daemon arguments (see `gkfs_daemon -h`)

DAEMON_ARGS=""

# use numactl to pin daemon to socket

DAEMON_ARGS="-l lo -c"

# use cpu affinity. Set this eg to `taskset -c ...`

DAEMON_AFFINITY=""



## proxy configuration

USE_PROXY=false

DAEMON_PROXY_ARGS="--proxy-listen lo --proxy-protocol ofi+sockets"

PROXY_LOCAL_PID_FILE=/dev/shm/vef_gkfs_proxy.pid

PROXY_ARGS="-p ofi+sockets"

# use cpu affinity. Set this eg to `taskset -c ...`

PROXY_AFFINITY=""



## slurm configuration

# Use Slurm's srun to start the daemons on multiple nodes and set specific srun args

USE_SRUN=false

# path to hostfile for srun for daemon

DAEMON_NODELIST=""

# srun args

SRUN_ARGS="--ntasks-per-node=1 --overcommit --contiguous --oversubscribe --mem=0"

DAEMON_NODELIST=./hostfile

PROXY_NODELIST=./hostfile

SRUN_ARGS="--overlap --ntasks-per-node=1 --overcommit --overlap --oversubscribe --mem=0"

# Specific srun args for daemon

SRUN_DAEMON_ARGS=""

# use cpu affinity. Set this eg to `taskset -c ...`

DAEMON_AFFINITY=""

# Specific srun args for proxy

SRUN_PROXY_ARGS=""

# path to daemon pid file; created where the script is run

SRUN_DAEMON_PID_FILE=./gkfs_daemon.pid

SRUN_PROXY_PID_FILE=./gkfs_proxy.pid



# logging

GKFS_DAEMON_LOG_LEVEL=info

GKFS_DAEMON_LOG_PATH=/dev/shm/vef_gkfs_daemon.log

LIBGKFS_LOG=errors,warnings

LIBGKFS_LOG_OUTPUT=/dev/shm/vef_gkfs_client.log
 
GKFS_DAEMON_LOG_PATH=/dev/shm/gkfs_daemon.log

GKFS_PROXY_LOG_LEVEL=info

GKFS_PROXY_LOG_PATH=/dev/shm/gkfs_proxy.log

# Modify the following for the client

# LIBGKFS_LOG=errors,warnings

# LIBGKFS_LOG_OUTPUT=/tmp/gkfs_client.log

scripts/run/gkfs_io500_proxy.conf→scripts/run/gkfs_io500.conf

+51 −0
Original line number Diff line number Diff line 
#!/bin/bash



# binaries (default for project_dir/build

PRELOAD_LIB=/lustre/miifs01/project/m2_zdvresearch/vef/io500/lib/libgkfs_intercept.so

DAEMON_BIN=/lustre/miifs01/project/m2_zdvresearch/vef/io500/bin/gkfs_daemon

PROXY_BIN=/lustre/miifs01/project/m2_zdvresearch/vef/io500/bin/gkfs_proxy

PRELOAD_LIB=/lustre/project/nhr-admire/vef/gekkofs/build/src/client/libgkfs_intercept.so

DAEMON_BIN=/lustre/project/nhr-admire/vef/gekkofs/build/src/daemon/gkfs_daemon

PROXY_BIN=/lustre/project/nhr-admire/vef/gekkofs/build/src/proxy/gkfs_proxy



## client configuration

LIBGKFS_HOSTS_FILE=/lustre/miifs01/project/m2_zdvresearch/vef/io500/run/gkfs_hostfile

LIBGKFS_HOSTS_FILE=/lustre/project/nhr-admire/vef/run/io500/gkfs_hostfile



## daemon configuration

DAEMON_ROOTDIR=/dev/shm/vef_gkfs_rootdir

#DAEMON_ROOTDIR=/dev/shm/vef_gkfs_rootdir

DAEMON_ROOTDIR=/localscratch/${SLURM_JOB_ID}/vef_gkfs_rootdir

#DAEMON_ROOTDIR=/localscratch/${SLURM_JOB_ID}/vef_gkfs_rootdir

DAEMON_MOUNTDIR=/dev/shm/vef_gkfs_mountdir

# additional daemon arguments (see `gkfs_daemon -h`)

# use numactl to pin daemon to socket

DAEMON_ARGS="-l ib0 -c"

DAEMON_ARGS="-P ofi+verbs -l ib0 -c"

# use cpu affinity. Set this eg to `taskset -c ...`

DAEMON_AFFINITY=""

DAEMON_AFFINITY="taskset -c 0-63"



## proxy configuration

DAEMON_PROXY_ARGS="--proxy-listen ib0 --proxy-protocol ofi+sockets"

USE_PROXY=false

DAEMON_PROXY_ARGS="--proxy-listen ib0 --proxy-protocol ofi+verbs"

PROXY_LOCAL_PID_FILE=/dev/shm/vef_gkfs_proxy.pid

PROXY_ARGS=""

PROXY_ARGS="-p ofi+verbs"

# use cpu affinity. Set this eg to `taskset -c ...`

PROXY_AFFINITY=""

PROXY_AFFINITY="taskset -c 0-63"



## slurm configuration

# Use Slurm's srun to start the daemons on multiple nodes and set specific srun args

USE_SRUN=true

# path to hostfile for srun for daemon and proxy

DAEMON_NODELIST=""

PROXY_NODELIST=""

# srun args

SRUN_ARGS="--ntasks-per-node=1 --overcommit --contiguous --oversubscribe --mem=0"

DAEMON_NODELIST=/lustre/project/nhr-admire/vef/hostfile

PROXY_NODELIST=/lustre/project/nhr-admire/vef/hostfile

SRUN_ARGS="--overlap --ntasks-per-node=1 --overcommit --overlap --oversubscribe --mem=0"

# Specific srun args for daemon

SRUN_DAEMON_ARGS=""

# Specific srun args for proxy

SRUN_PROXY_ARGS=""

# path to daemon pid file; created where the script is run

SRUN_DAEMON_PID_FILE=/lustre/miifs01/project/m2_zdvresearch/vef/io500/run/gkfs_daemon.pid

SRUN_PROXY_PID_FILE=/lustre/miifs01/project/m2_zdvresearch/vef/io500/run/gkfs_proxy.pid

SRUN_DAEMON_PID_FILE=/lustre/project/nhr-admire/vef/run/io500/gkfs_daemon.pid

SRUN_PROXY_PID_FILE=/lustre/project/nhr-admire/vef/run/io500/gkfs_proxy.pid



# logging configuration

GKFS_DAEMON_LOG_LEVEL=info

GKFS_DAEMON_LOG_PATH=/dev/shm/vef_gkfs_daemon.log

GKFS_PROXY_LOG_LEVEL=info

GKFS_PROXY_LOG_PATH=/dev/shm/vef_gkfs_proxy.log

LIBGKFS_LOG=errors,warnings

LIBGKFS_LOG_OUTPUT=/dev/shm/vef_gkfs_client.log
 
 No newline at end of file
 
# Modify the following for the client

# LIBGKFS_LOG=errors,warnings

# LIBGKFS_LOG_OUTPUT=/dev/shm/vef_gkfs_client.log
 
 No newline at end of file