diff --git a/plugins/slurm/defaults.h.in b/plugins/slurm/defaults.h.in index 9de23bc60bdbf9b9bacde3be859b1928c1cad93d..7dc3b7c3530f7a0451def5b2fbebee28814ebeef 100644 --- a/plugins/slurm/defaults.h.in +++ b/plugins/slurm/defaults.h.in @@ -39,7 +39,7 @@ #define SCORDCTL_TMPDIR_DEFAULT "/tmp" #define CARGO_PROG_DEFAULT "@CARGO_PROGRAM@" #define CARGO_PROTO_DEFAULT SCORD_PROTO_DEFAULT -#define CARGO_PORT_DEFAULT 62000 +#define CARGO_PORT_DEFAULT @CARGO_BIND_PORT@ // clang-format on diff --git a/plugins/slurm/scord_common.sh.in b/plugins/slurm/scord_common.sh.in index f67242b3a5403903c895e4fdff0df9d026479d3c..ed4eff01e2161435285edb0d5fe96114652c7288 100644 --- a/plugins/slurm/scord_common.sh.in +++ b/plugins/slurm/scord_common.sh.in @@ -183,4 +183,4 @@ export SCORD_QUERY_PROGRAM="@SCORD_QUERY_PROGRAM@" export SCORDCTL_PROGRAM="@SCORDCTL_PROGRAM@" export SCORDCTL_PROTO="@SCORD_TRANSPORT_PROTOCOL@" export SCORDCTL_PORT="@SCORD_CTL_BIND_PORT@" -export CARGO_PORT="@CARGO_PORT@" +export CARGO_PORT="@CARGO_BIND_PORT@" diff --git a/plugins/slurm/scord_prolog.sh.in b/plugins/slurm/scord_prolog.sh.in index 8389cf148e882a3a5835369e83437225414d5d82..d67e0ececfeddba99a7dd5ffea83f34e8e957ba8 100755 --- a/plugins/slurm/scord_prolog.sh.in +++ b/plugins/slurm/scord_prolog.sh.in @@ -143,7 +143,7 @@ fi # stored (note that $HOME is not set when this prolog script is being executed). # shellcheck disable=SC2016 USER_HOME=$(run_as "$SLURM_JOB_USER" echo '$HOME') -USER_CONFIG_DIRECTORY="${XDG_CONFIG_HOME:-$USER_HOME/.config}" +USER_CONFIG_DIRECTORY="$USER_HOME/.config" CARGO_CONFIG_DIRECTORY="$USER_CONFIG_DIRECTORY/cargo" SYSTEMD_USER_DIRECTORY="$USER_CONFIG_DIRECTORY/systemd/user" @@ -189,16 +189,16 @@ CARGO_HOSTS=$hostnames_csv CARGO_NUM_NODES=$CARGO_NUM_NODES CARGO_ADDRESS=$CARGO_MASTER_ADDRESS EOT - +CUID=$(id -u $SLURM_JOB_USER) chown "$SLURM_JOB_USER":"$SLURM_JOB_GROUP" "$CARGO_CONFIG_FILE" -if ! run_as "$SLURM_JOB_USER" systemctl --user start "$CARGO_INSTANCE_NAME"; then +if ! run_as "$SLURM_JOB_USER" DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$CUID/bus systemctl --user start "$CARGO_INSTANCE_NAME"; then exit 1 fi sleep 1s -if ! run_as "$SLURM_JOB_USER" systemctl --user is-active --quiet "$CARGO_INSTANCE_NAME"; then +if ! run_as "$SLURM_JOB_USER" DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$CUID/bus systemctl --user is-active --quiet "$CARGO_INSTANCE_NAME"; then echo "Cargo data stager failed to start" exit 1 fi diff --git a/plugins/slurm/systemd/cargo@.service.in b/plugins/slurm/systemd/cargo@.service.in index a04872b8a6b35a5304d66c527989f58aa36ad411..b0bcb46b997224be8b8131e3d615b4fb1dbc166d 100644 --- a/plugins/slurm/systemd/cargo@.service.in +++ b/plugins/slurm/systemd/cargo@.service.in @@ -3,7 +3,7 @@ Description=Cargo parallel data stager [Service] Type=simple -EnvironmentFile=%S/cargo/%I.cfg +EnvironmentFile=%h/.config/cargo/%I.cfg ExecStart=@CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/slurm/cargoctl start -s ${CARGO_ADDRESS} -H ${CARGO_HOSTS} -n ${CARGO_NUM_NODES} ExecStop=@CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/slurm/cargoctl stop -s ${CARGO_ADDRESS} Restart=no diff --git a/src/scord/rpc_server.cpp b/src/scord/rpc_server.cpp index d706b3349f339ec46721505189803b69eaf51315..da3027a350facdd877b58c0cb92c7b78c69462ee 100644 --- a/src/scord/rpc_server.cpp +++ b/src/scord/rpc_server.cpp @@ -951,7 +951,8 @@ rpc_server::scheduler_update() { std::vector> return_set; const auto threshold = 0.1f; while(!m_shutting_down) { - thallium::thread::self().sleep(m_network_engine, 1000); + sleep(1); + //thallium::thread::self().sleep(m_network_engine, 500); m_transfer_manager.lock(); const auto transfer = m_transfer_manager.transfer(); std::vector v_ids; diff --git a/src/scord/rpc_server.hpp b/src/scord/rpc_server.hpp index 33f4d01b86245c09da090f2b774cd73365b06ea0..506292838f9b563bbf4d0d412d83a2a37f2f4c31 100644 --- a/src/scord/rpc_server.hpp +++ b/src/scord/rpc_server.hpp @@ -118,8 +118,8 @@ private: adhoc_storage_manager m_adhoc_manager; pfs_storage_manager m_pfs_manager; transfer_manager m_transfer_manager; - - // Dedicated execution stream for the MPI listener ULT + + // Dedicated execution stream for the Scheduler listener ULT thallium::managed m_scheduler_ess; // ULT for the MPI listener thallium::managed m_scheduler_ult;