diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fef13ef2d2249f6cb2e84092c31acb1d14f8da6a..d3d4ebd8d535f22a7400667353d0f026173c75fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,69 @@ -# You can override the included template(s) by including variable overrides -# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings -# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings -# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings -# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings -# Note that environment variables can be set in several places -# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence +image: bscstorage/cargo:0.2.0-wip + stages: -- test + - build + - test + +variables: + PREFIX: + /usr/local + LD_LIBRARY_PATH: >- + /usr/lib/: + /usr/lib64: + /usr/local/lib: + /usr/local/lib64 + PKG_CONFIG_PATH: >- + /usr/lib/pkgconfig: + /usr/lib64/pkgconfig: + /usr/local/lib/pkgconfig: + /usr/local/lib64/pkgconfig: + /usr/lib64/openmpi/lib/pkgconfig + +before_script: + - source /etc/profile.d/modules.sh + - module load mpi + +release: + stage: build + script: + - cmake --preset ci-release + - cmake --build builds/ci-release -j$(nproc) --target install + +debug: + stage: build + script: + - cmake --preset ci-debug + - cmake --build builds/ci-debug -j$(nproc) --target install + - cd builds/ci-debug + # cleanup intermediate files to save on artifact space + - grep "^rule.*\(_COMPILER_\|_STATIC_LIBRARY_\)" + $(find . -name rules.ninja) | + cut -d ' ' -f2 | + xargs -n1 ninja -t clean -r + artifacts: + expire_in: 2 days + paths: + - builds/ci-debug + +integration: + stage: test + needs: [ debug ] + variables: + OMPI_ALLOW_RUN_AS_ROOT: "1" + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: "1" + script: + - ctest --test-dir builds/ci-debug -j$(nproc) --output-junit report.xml + + artifacts: + expire_in: 1 week + paths: + - builds/ci-debug/Testing/Temporary + reports: + junit: builds/ci-debug/report.xml + sast: stage: test + before_script: [] + needs: [] include: -- template: Security/SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e61830be99c581eb239a8cdbf831b6c9687675d..536711b1038491fb4d7d898c11471195bbc98e50 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,9 +141,9 @@ message(STATUS "[${PROJECT_NAME}] server bind address: ${CARGO_BIND_ADDRESS}") ### server bind port set(CARGO_BIND_PORT - "52000" + "62000" CACHE STRING - "Define the bind port for the ${PROJECT_NAME} server (default: 52000)" + "Define the bind port for the ${PROJECT_NAME} server (default: 62000)" ) message(STATUS "[${PROJECT_NAME}] server bind port: ${CARGO_BIND_PORT}") @@ -322,6 +322,7 @@ endif () add_subdirectory(etc) add_subdirectory(lib) add_subdirectory(src) +add_subdirectory(util) if(CARGO_BUILD_TESTS) add_subdirectory(tests) diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000000000000000000000000000000000000..15acccd390d799a0af8e75ec42b3ce93cef26c3d --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,203 @@ +{ + "version": 2, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "base", + "displayName": "GCC", + "description": "Sets prefix, build, and install directories as well as common options", + "hidden": true, + "generator": "Ninja", + "binaryDir": "${sourceDir}/builds/${presetName}", + "cacheVariables": { + "CMAKE_CXX_COMPILER_LAUNCHER": "/usr/bin/ccache", + "CMAKE_C_COMPILER_LAUNCHER": "/usr/bin/ccache", + "CMAKE_PREFIX_PATH": "${sourceParentDir}/prefix", + "CMAKE_INSTALL_PREFIX": "${sourceParentDir}/prefix", + "CARGO_BUILD_TESTS": true + } + }, + { + "name": "debug", + "displayName": "Debug", + "description": "Build options for Debug", + "hidden": true, + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_CXX_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always --pedantic" + } + }, + { + "name": "release", + "displayName": "Releae", + "description": "Build options for Release", + "hidden": true, + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "gcc", + "displayName": "GCC (system default)", + "description": "Build options for GCC (system default)", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/g++", + "CMAKE_C_COMPILER": "/usr/bin/gcc", + "CMAKE_CXX_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always", + "CMAKE_C_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always" + } + }, + { + "name": "gcc-debug", + "displayName": "GCC (system default, debug)", + "description": "Build options for GCC (system default)", + "inherits": "gcc", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_C_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always -O0" + } + }, + { + "name": "gcc-10", + "displayName": "GCC 10", + "description": "Build options for GCC 10", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/g++-10", + "CMAKE_C_COMPILER": "/usr/bin/gcc-10", + "CMAKE_CXX_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always", + "CMAKE_C_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always" + } + }, + { + "name": "gcc-11", + "displayName": "GCC 11", + "description": "Build options for GCC 11", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/g++-11", + "CMAKE_C_COMPILER": "/usr/bin/gcc-11", + "CMAKE_CXX_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always", + "CMAKE_C_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always" + } + }, + { + "name": "clang", + "displayName": "Clang (system default)", + "description": "Build options for Clang (system default)", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/clang++", + "CMAKE_CXX_FLAGS": "-fdiagnostics-color=always", + "CMAKE_C_COMPILER": "/usr/bin/clang", + "CMAKE_C_FLAGS": "-Wno-unused-command-line-argument -fdiagnostics-color=always" + } + }, + { + "name": "clang-10", + "displayName": "Clang 10", + "description": "Build options for Clang 10", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/clang++-10", + "CMAKE_CXX_FLAGS": "-fdiagnostics-color=always", + "CMAKE_C_COMPILER": "/usr/bin/clang-10", + "CMAKE_C_FLAGS": "-Wno-unused-command-line-argument -fdiagnostics-color=always" + } + }, + { + "name": "clang-11", + "displayName": "Clang 11", + "description": "Build options for Clang 11", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/clang++-11", + "CMAKE_CXX_FLAGS": "-fdiagnostics-color=always", + "CMAKE_C_COMPILER": "/usr/bin/clang-11", + "CMAKE_C_FLAGS": "-Wno-unused-command-line-argument -fdiagnostics-color=always" + } + }, + { + "name": "clang-12", + "displayName": "Clang 12", + "description": "Build options for Clang 12", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/clang++-12", + "CMAKE_CXX_FLAGS": "-fdiagnostics-color=always", + "CMAKE_C_COMPILER": "/usr/bin/clang-12", + "CMAKE_C_FLAGS": "-Wno-unused-command-line-argument -fdiagnostics-color=always" + } + }, + { + "name": "ci", + "displayName": "CI", + "description": "Build options for CI", + "inherits": "base", + "hidden": true, + "cacheVariables": { + "CMAKE_CXX_COMPILER_LAUNCHER": null, + "CMAKE_C_COMPILER_LAUNCHER": null, + "CMAKE_C_COMPILER": "/usr/bin/gcc", + "CMAKE_CXX_COMPILER": "/usr/bin/g++", + "CMAKE_INSTALL_PREFIX": "/usr/local", + "CMAKE_PREFIX_PATH": "/usr/local", + "Boost_LIBRARY_DIR": "/usr/lib;/usr/lib64;/usr/lib64/openmpi/lib", + "CARGO_TRANSPORT_LIBRARY": "libfabric", + "CARGO_TRANSPORT_PROTOCOL": "ofi+tcp", + "CARGO_BIND_ADDRESS": "127.0.0.1", + "CARGO_BIND_PORT": "62000", + "CARGO_BUILD_TESTS": true + } + }, + { + "name": "ci-debug", + "displayName": "CI (debug)", + "description": "Build options for CI (debug)", + "inherits": ["ci", "debug"] + }, + { + "name": "ci-release", + "displayName": "CI (debug)", + "description": "Build options for CI (debug)", + "inherits": ["ci", "release"] + } + ], + "buildPresets": [ + { + "name": "core-build", + "description": "Inherits environment from base configurePreset", + "configurePreset": "base", + "hidden": true + }, + { + "name": "gcc", + "description": "Build with default GCC", + "configurePreset": "gcc", + "inherits": "core-build" + }, + { + "name": "gcc-11", + "description": "Build with GCC 11", + "configurePreset": "gcc-11", + "inherits": "core-build" + }, + { + "name": "clang", + "description": "Build with default Clang", + "configurePreset": "clang", + "inherits": "core-build" + }, + { + "name": "clang-10", + "description": "Build with Clang 10", + "configurePreset": "clang-10", + "inherits": "core-build" + } + ] +} diff --git a/README.md b/README.md index f93ac25afc21ee4dd71deb7478d0562a518ef894..1fccc2dfea3eac1b1fbb0bd5047759b9c83feee7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,14 @@ -# Cargo +
+

Cargo

+ +[![build status)](https://img.shields.io/gitlab/pipeline-status/hpc/cargo?gitlab_url=https%3A%2F%2Fstorage.bsc.es%2Fgitlab%2F&logo=gitlab)](https://storage.bsc.es/gitlab/hpc/cargo/-/pipelines) +[![latest release](https://storage.bsc.es/gitlab/hpc/cargo/-/badges/release.svg)](https://storage.bsc.es/gitlab/hpc/cargo/-/releases) +[![GitLab (self-managed)](https://img.shields.io/gitlab/license/hpc/cargo?gitlab_url=https%3A%2F%2Fstorage.bsc.es%2Fgitlab)](https://storage.bsc.es/gitlab/hpc/cargo/-/blob/main/COPYING) +[![Language](https://img.shields.io/static/v1?label=language&message=C99%20%2F%20C%2B%2B20&color=red)](https://en.wikipedia.org/wiki/C%2B%2B20) + +

A parallel data staging service for HPC clusters

+ +
Cargo is a HPC data staging service that runs alongside applications helping them to transfer data in parallel between local and shared storage tiers. @@ -137,8 +147,26 @@ library (`${INSTALL_DIR}/lib/libcargo.so`) and its headers ## Testing +Tests can be run automatically with CTest: + +```shell +cd build +ctest -VV --output-on-failure --stop-on-failure -j 8 +``` + +When this happens, a Cargo server with 3 workers is automatically started +(via `mpirun`/`mpiexec`) and stopped (via RPC) so that tests can progress. + +Alternatively, during development one may desire to run the Cargo server +manually and then the tests. In this case, the following commands can be used: + ```shell -cd build/tests/ -mpirun -np 4 ${INSTALL_DIR}/bin/cargo -C -./tests -S ofi+tcp://127.0.0.1:52000 +# start the Cargo server with 3 workers. The server will be listening on +# port 62000 and will communicate with workers via MPI messages. The server can +# be stopped with Ctrl+C, `kill -TERM ` or `cargo_shutdown
`.) +mpirun -np 4 ${INSTALL_DIR}/bin/cargo -l ofi+tcp://127.0.0.1:62000 + +# run the tests +cd build +RUNNER_SKIP_START=1 ctest -VV --output-on-failure --stop-on-failure -j 8 ``` diff --git a/docker/0.2.0-wip/Dockerfile b/docker/0.2.0-wip/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..53b3b4e299136ebaac41c52f3bc3a400eed571be --- /dev/null +++ b/docker/0.2.0-wip/Dockerfile @@ -0,0 +1,133 @@ +FROM rockylinux:9.2 + +RUN set -ex \ + && yum makecache \ + && yum -y update \ + && yum -y install dnf-plugins-core \ + && yum config-manager --set-enabled crb \ + && yum -y install \ + gcc \ + gcc-c++\ + gdb \ + git \ + gnupg \ + make \ + automake \ + libtool \ + file \ + ninja-build \ + json-c-devel \ + libibverbs-devel \ + boost-devel \ + boost-openmpi-devel \ + json-c-devel \ + openmpi-devel \ + libconfig-devel \ + # install cmake 3.21+ since we need to produce JUnit XML files + && curl -OL https://github.com/Kitware/CMake/releases/download/v3.27.6/cmake-3.27.6-Linux-x86_64.sh \ + && chmod u+x ./cmake-3.27.6-Linux-x86_64.sh \ + && ./cmake-3.27.6-Linux-x86_64.sh --skip-license --prefix=/usr \ + # cleanup + && yum clean all \ + && rm -rf /var/cache/yum \ + && rm ./cmake-3.27.6-Linux-x86_64.sh + +# Download and install dependencies +RUN set -ex \ + && export LD_LIBRARY_PATH=${DEPS_INSTALL_PATH}/lib:${DEPS_INSTALL_PATH}/lib64 \ + && export PKG_CONFIG_PATH=${DEPS_INSTALL_PATH}/lib/pkgconfig:${DEPS_INSTALL_PATH}/lib64/pkgconfig \ + && cd \ + && mkdir deps \ + && cd deps \ + && git clone https://github.com/ofiwg/libfabric --recurse-submodules \ + && git clone https://github.com/pmodels/argobots --recurse-submodules \ + && git clone https://github.com/mercury-hpc/mercury --recurse-submodules \ + && git clone https://github.com/mochi-hpc/mochi-margo --recurse-submodules \ + && git clone https://github.com/USCiLab/cereal --recurse-submodules \ + && git clone https://github.com/mochi-hpc/mochi-thallium --recurse-submodules \ + \ + && cd \ + ### argobots + && cd deps/argobots \ + && ./autogen.sh \ + && mkdir build \ + && cd build \ + && CFLAGS="-ggdb3 -O0" ../configure --prefix=${DEPS_INSTALL_PATH} \ + && make install -j \ + && cd .. \ + && rm -rf build \ + && cd \ + \ + ### libfabric + && cd deps/libfabric \ + && git checkout v1.14.0rc3 \ + && ./autogen.sh \ + && mkdir build \ + && cd build \ + && CFLAGS="-ggdb3 -O0" ../configure --prefix=${DEPS_INSTALL_PATH} \ + && make install -j \ + && cd .. \ + && rm -rf build \ + && cd \ + \ + ### mercury + && cd deps/mercury \ + && mkdir build && cd build \ + && cmake \ + -DMERCURY_USE_SELF_FORWARD:BOOL=ON \ + -DBUILD_TESTING:BOOL=ON \ + -DMERCURY_USE_BOOST_PP:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DNA_USE_OFI:BOOL=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + .. \ + && make install -j \ + && cd .. \ + && rm -rf build \ + && cd \ + \ + ### mochi-margo + && cd deps/mochi-margo \ + && ./prepare.sh \ + && mkdir build \ + && cd build \ + && CFLAGS="-ggdb3 -O0" ../configure --prefix=${DEPS_INSTALL_PATH} \ + && make -j install \ + && cd .. \ + && rm -rf build \ + && cd \ + \ + ### cereal + && cd deps/cereal \ + && mkdir build \ + && cd build \ + \ + && cmake \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + -DBUILD_DOC:BOOL=OFF \ + -DBUILD_SANDBOX:BOOL=OFF \ + -DBUILD_TESTS:BOOL=OFF \ + -DSKIP_PERFORMANCE_COMPARISON:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \ + .. \ + && make -j install \ + && cd .. \ + && rm -rf build \ + && cd \ + \ + ### mochi-thallium + && cd deps/mochi-thallium \ + && mkdir build \ + && cd build \ + && cmake \ + -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + .. \ + && make -j install \ + && cd .. \ + && rm -rf build \ + && cd \ + \ + && rm -rf deps diff --git a/scripts/runner.sh b/scripts/runner.sh new file mode 100755 index 0000000000000000000000000000000000000000..97701bc98fc2ed9037a3a3beb5402238996f9495 --- /dev/null +++ b/scripts/runner.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +function help() { + echo "Usage:" + echo " $(basename "$0") COMMAND" + echo "" + echo " Where COMMAND is one of:" + echo " start PIDFILE PROGRAM ARGS... Run PROGRAM and record its PID in PIDFILE" + echo " stop SIGNAL PIDFILE Send SIGNAL to the PID contained in PIDFILE" + echo " help Print this message" +} + +function readpid() { + + if [ $# -eq 0 ]; then + echo "FATAL: readpid(): Missing pifile" >&2 + exit 1 + fi + + pidfile="$1" + + read -r pid <"$pidfile" + echo $pid +} + +function run() { + + if [[ -n "${RUNNER_SKIP_START}" && "${RUNNER_SKIP_START}" != "0" ]]; then + exit 0 + fi + + if [ $# -eq 0 ]; then + echo "FATAL: missing program pidfile" >&2 + elif [ $# -eq 1 ]; then + echo "FATAL: missing program to run" >&2 + help + exit 1 + fi + + pidfile="$1" + shift + + if [ -e "$pidfile" ]; then + pid=$(readpid "$pidfile") + echo "$pid" + + if pgrep --pidfile "$pidfile"; then + exit 1 + fi + fi + + "$@" 2>runner.$$.err 1>runner.$$.out 0"$pidfile" + sleep 1 + + if ! kill -0 $pid; then + echo "Process $pid does not seem to exist." >&2 + echo "The program below may not exist or may have crashed while starting:" >&2 + echo " $*" >&2 + echo " STDOUT: " >&2 + cat "runner.$$.out" >&2 + echo " STDERR: " >&2 + cat "runner.$$.err" >&2 + rm runner.$$.out runner.$$.err + exit 1 + fi + + exit 0 +} + +function stop() { + + if [ $# -eq 0 ]; then + echo "FATAL: missing signal" >&2 + exit 1 + elif [ $# -eq 1 ]; then + echo "FATAL: missing pidfile" >&2 + exit 1 + fi + + signal="$1" + pidfile="$2" + + if [ ! -e "$pidfile" ]; then + echo "FATAL: pidfile '$pidfile' does not exist" >&2 + exit 1 + fi + + if pkill "-$signal" --pidfile "$pidfile"; then + rm "$pidfile" + rm runner.$$.out runner.$$.err + exit 0 + fi + exit 1 +} + +if [ $# -eq 0 ]; then + echo "FATAL: missing arguments" >&2 + help + exit 1 +fi + +case $1 in + +start) + shift + run "$@" + ;; + +stop) + shift + stop "$@" + ;; + +help) + help + exit 0 + ;; +esac diff --git a/src/master.cpp b/src/master.cpp index e274eb2b28b6afa0ecb4fda2c823aee232d6c005..18953e1d7946b75b79fd5a3a66b74711a376a09b 100644 --- a/src/master.cpp +++ b/src/master.cpp @@ -82,6 +82,7 @@ master_server::master_server(std::string name, std::string address, #define EXPAND(rpc_name) #rpc_name##s, &master_server::rpc_name provider::define(EXPAND(ping)); + provider::define(EXPAND(shutdown)); provider::define(EXPAND(transfer_datasets)); provider::define(EXPAND(transfer_status)); @@ -161,6 +162,17 @@ master_server::ping(const network::request& req) { req.respond(resp); } +void master_server::shutdown(const network::request& req) { + using network::get_address; + using network::rpc_info; + using proto::generic_response; + + const auto rpc = rpc_info::create(RPC_NAME(), get_address(req)); + + LOGGER_INFO("rpc {:>} body: {{}}", rpc); + server::shutdown(); +} + void master_server::transfer_datasets(const network::request& req, const std::vector& sources, diff --git a/src/master.hpp b/src/master.hpp index 77ca0fd4236ba88c3fe474c59813a6e080fe76d6..3d5797ac8919965f8451138357890f9f382b50c4 100644 --- a/src/master.hpp +++ b/src/master.hpp @@ -47,6 +47,9 @@ private: void ping(const network::request& req); + void + shutdown(const network::request& req); + void transfer_datasets(const network::request& req, const std::vector& sources, diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt index 056dcf510672c2bcdf2d26bf8277f32f762f607a..5aa4c9d576eed614174b42c2386c2045dbd9f6f2 100644 --- a/src/net/CMakeLists.txt +++ b/src/net/CMakeLists.txt @@ -31,6 +31,7 @@ target_sources( ) target_link_libraries(rpc_common PUBLIC logger::logger thallium asio::asio) +set_property(TARGET rpc_common PROPERTY POSITION_INDEPENDENT_CODE ON) # get the parent directory of the current directory so we can include # headers from these libraries as `` diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 99998abfb284b6114765264203ef3fe892fac0ee..72bb98e93d779e9abcd7ebfff22c0e654035ddba 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -34,7 +34,35 @@ target_link_libraries( tests PUBLIC Catch2::Catch2 Boost::iostreams fmt::fmt cargo posix_file ) +# prepare the environment for the Cargo daemon +set(CARGO_TESTS_DIRECTORY "${CMAKE_BINARY_DIR}/Testing") +file(MAKE_DIRECTORY ${CARGO_TESTS_DIRECTORY}) + +set(TEST_DIRECTORY "${CARGO_TESTS_DIRECTORY}/cargo_server") +file(MAKE_DIRECTORY ${TEST_DIRECTORY}) + +set(CARGO_ADDRESS + ${CARGO_TRANSPORT_PROTOCOL}://${CARGO_BIND_ADDRESS}:${CARGO_BIND_PORT}) + +add_test(NAME start_cargo + COMMAND + ${CMAKE_SOURCE_DIR}/scripts/runner.sh start /dev/null + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 + $ -l ${CARGO_ADDRESS} -o ${TEST_DIRECTORY}/cargo.log +) + +set_tests_properties(start_cargo + PROPERTIES FIXTURES_SETUP cargo_daemon) + +add_test(NAME stop_cargo + COMMAND cargo_shutdown --server ${CARGO_ADDRESS} +) + +set_tests_properties(stop_cargo + PROPERTIES FIXTURES_CLEANUP cargo_daemon) + catch_discover_tests( tests EXTRA_ARGS "-S ${CARGO_TRANSPORT_PROTOCOL}://${CARGO_BIND_ADDRESS}:${CARGO_BIND_PORT}" + PROPERTIES FIXTURES_REQUIRED cargo_daemon ) diff --git a/tests/tests.cpp b/tests/tests.cpp index 15b682bb5a2e17546aa9dd1ef1934258b86525d6..6ec9d742c7b90f22b420b7b8f840c24559481883 100644 --- a/tests/tests.cpp +++ b/tests/tests.cpp @@ -215,10 +215,11 @@ SCENARIO("Parallel reads", "[flex_stager][parallel_reads]") { cargo::server server{server_address}; - const auto sources = prepare_datasets(cargo::dataset::type::parallel, - "source-dataset-{}", NDATASETS); - const auto targets = prepare_datasets(cargo::dataset::type::posix, - "target-dataset-{}", NDATASETS); + const auto sources = + prepare_datasets(cargo::dataset::type::parallel, + "pr-source-dataset-{}", NDATASETS); + const auto targets = prepare_datasets( + cargo::dataset::type::posix, "pr-target-dataset-{}", NDATASETS); static std::vector input_files; input_files.reserve(sources.size()); @@ -275,10 +276,11 @@ SCENARIO("Parallel writes", "[flex_stager][parallel_writes]") { cargo::server server{server_address}; - const auto sources = prepare_datasets(cargo::dataset::type::posix, - "source-dataset-{}", NDATASETS); - const auto targets = prepare_datasets(cargo::dataset::type::parallel, - "target-dataset-{}", NDATASETS); + const auto sources = prepare_datasets( + cargo::dataset::type::posix, "pw-source-dataset-{}", NDATASETS); + const auto targets = + prepare_datasets(cargo::dataset::type::parallel, + "pw-target-dataset-{}", NDATASETS); static std::vector input_files; input_files.reserve(sources.size()); diff --git a/util/CMakeLists.txt b/util/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..a1ee38475250e0f4089d9e2535c240390feda6cd --- /dev/null +++ b/util/CMakeLists.txt @@ -0,0 +1,57 @@ +################################################################################ +# Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain # +# # +# This software was partially supported by the EuroHPC-funded project ADMIRE # +# (Project ID: 956748, https://www.admire-eurohpc.eu). # +# # +# This file is part of Cargo. # +# # +# Cargo is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# Cargo is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with Cargo. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +add_executable(cargo_ping) + +target_sources(cargo_ping + PRIVATE + ping.cpp +) + +target_link_libraries(cargo_ping + PUBLIC + fmt::fmt + CLI11::CLI11 + net::rpc_client + cargo +) + +add_executable(cargo_shutdown) + +target_sources(cargo_shutdown + PRIVATE + shutdown.cpp +) + +target_link_libraries(cargo_shutdown + PUBLIC + fmt::fmt + CLI11::CLI11 + net::rpc_client + cargo +) + +install(TARGETS cargo_ping cargo_shutdown + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} +) diff --git a/util/ping.cpp b/util/ping.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2449f8b70e8b2b5268b2189d93779a6347b4bf25 --- /dev/null +++ b/util/ping.cpp @@ -0,0 +1,103 @@ +/****************************************************************************** + * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of Cargo. + * + * Cargo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Cargo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Cargo. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include + +struct ping_config { + std::string progname; + std::string server_address; +}; + +ping_config +parse_command_line(int argc, char* argv[]) { + + ping_config cfg; + + cfg.progname = std::filesystem::path{argv[0]}.filename().string(); + + CLI::App app{"Cargo ping client", cfg.progname}; + + app.add_option("-s,--server", cfg.server_address, "Server address") + ->option_text("ADDRESS") + ->required(); + + try { + app.parse(argc, argv); + return cfg; + } catch(const CLI::ParseError& ex) { + std::exit(app.exit(ex)); + } +} + +auto +parse_address(const std::string& address) { + const auto pos = address.find("://"); + if(pos == std::string::npos) { + throw std::runtime_error(fmt::format("Invalid address: {}", address)); + } + + const auto protocol = address.substr(0, pos); + return std::make_pair(protocol, address); +} + + +int +main(int argc, char* argv[]) { + + ping_config cfg = parse_command_line(argc, argv); + + try { + const auto [protocol, address] = parse_address(cfg.server_address); + network::client rpc_client{protocol}; + + if(const auto result = rpc_client.lookup(address); result.has_value()) { + const auto& endpoint = result.value(); + const auto retval = endpoint.call("ping"); + + if(retval.has_value()) { + + auto error_code = int{retval.value()}; + + fmt::print("ping RPC was successful!\n"); + fmt::print(" (server replied with: {})\n", error_code); + return EXIT_SUCCESS; + } + + fmt::print(stderr, "ping RPC failed\n"); + return EXIT_FAILURE; + + } else { + fmt::print(stderr, "Failed to lookup address: {}\n", address); + return EXIT_FAILURE; + } + } catch(const std::exception& ex) { + fmt::print(stderr, "Error: {}\n", ex.what()); + return EXIT_FAILURE; + } +} diff --git a/util/shutdown.cpp b/util/shutdown.cpp new file mode 100644 index 0000000000000000000000000000000000000000..89b6d9886241625970e44a78f8d1801316ce3ae7 --- /dev/null +++ b/util/shutdown.cpp @@ -0,0 +1,92 @@ +/****************************************************************************** + * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of Cargo. + * + * Cargo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Cargo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Cargo. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include + +struct ping_config { + std::string progname; + std::string server_address; +}; + +ping_config +parse_command_line(int argc, char* argv[]) { + + ping_config cfg; + + cfg.progname = std::filesystem::path{argv[0]}.filename().string(); + + CLI::App app{"Cargo shutdown client", cfg.progname}; + + app.add_option("-s,--server", cfg.server_address, "Server address") + ->option_text("ADDRESS") + ->required(); + + try { + app.parse(argc, argv); + return cfg; + } catch(const CLI::ParseError& ex) { + std::exit(app.exit(ex)); + } +} + +auto +parse_address(const std::string& address) { + const auto pos = address.find("://"); + if(pos == std::string::npos) { + throw std::runtime_error(fmt::format("Invalid address: {}", address)); + } + + const auto protocol = address.substr(0, pos); + return std::make_pair(protocol, address); +} + + +int +main(int argc, char* argv[]) { + + ping_config cfg = parse_command_line(argc, argv); + + try { + const auto [protocol, address] = parse_address(cfg.server_address); + network::client rpc_client{protocol}; + + if(const auto result = rpc_client.lookup(address); result.has_value()) { + const auto& endpoint = result.value(); + endpoint.call("shutdown"); + fmt::print(stdout, "shutdown RPC sent to {}\n", address); + return EXIT_SUCCESS; + } + + fmt::print(stderr, "Failed to lookup address: {}\n", address); + return EXIT_FAILURE; + } catch(const std::exception& ex) { + fmt::print(stderr, "Error: {}\n", ex.what()); + return EXIT_FAILURE; + } +}