Commit cd76bc08 authored by Ramon Nou's avatar Ramon Nou
Browse files

Wait for active GekkoFS hosts with calibration

parent 6d3d216f

scripts/run/gkfs_run

+16 −3
Changes for scripts/run/gkfs_run: 16 added lines, 3 removed lines.
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 
export GKFS_MNT=${GKFS_MNT:-/dev/shm/gkfs_mnt}

export GKFS_ROOT=${GKFS_ROOT:-/dev/shm/gkfs_root}

export GKFS_DAEMON_LOG_LEVEL=${GKFS_DAEMON_LOG_LEVEL:-0}

export GKFS_HOSTS_FILE_TIMEOUT=${GKFS_HOSTS_FILE_TIMEOUT:-600}



# Helper function to print usage

usage() {
@@ -52,17 +53,29 @@ start_servers() { 
         -c 112 --overlap --overcommit --mem=0 --oversubscribe --export="ALL" \

         /bin/bash -c "${CMD}" &

    

    # Wait for servers to be ready

    # Wait for servers to be ready. Calibration adds comment lines to the

    # hostfile, so count only non-empty, non-comment daemon entries.

    echo "Waiting for servers to start..."

    WAITED=0

    while true; do

        if [ -f "$GKFS_HOSTS_FILE" ]; then

            LINE_COUNT=$(wc -l < "$GKFS_HOSTS_FILE")

            if [ "$LINE_COUNT" -ge "${SLURM_JOB_NUM_NODES}" ]; then

            HOST_COUNT=$(awk '

                /^[[:space:]]*#/ { next }

                /^[[:space:]]*$/ { next }

                { count++ }

                END { print count + 0 }

            ' "$GKFS_HOSTS_FILE")

            if [ "$HOST_COUNT" -ge "${SLURM_JOB_NUM_NODES}" ]; then

                echo "All ${SLURM_JOB_NUM_NODES} servers started."

                break

            fi

        fi

        if [ "$WAITED" -ge "$GKFS_HOSTS_FILE_TIMEOUT" ]; then

            echo "Error: timed out after ${GKFS_HOSTS_FILE_TIMEOUT}s waiting for ${SLURM_JOB_NUM_NODES} servers (found ${HOST_COUNT:-0} host entries)."

            return 1

        fi

        sleep 1

        WAITED=$((WAITED + 1))

    done

    

    echo "GekkoFS servers are ready."