From 708c63b026707ad612973f41c820c5a64beea67b Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 28 Sep 2023 12:21:59 +0200 Subject: [PATCH 01/10] Add `scripts/runner.sh` as program starter for tests --- scripts/runner.sh | 121 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100755 scripts/runner.sh diff --git a/scripts/runner.sh b/scripts/runner.sh new file mode 100755 index 0000000..97701bc --- /dev/null +++ b/scripts/runner.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +function help() { + echo "Usage:" + echo " $(basename "$0") COMMAND" + echo "" + echo " Where COMMAND is one of:" + echo " start PIDFILE PROGRAM ARGS... Run PROGRAM and record its PID in PIDFILE" + echo " stop SIGNAL PIDFILE Send SIGNAL to the PID contained in PIDFILE" + echo " help Print this message" +} + +function readpid() { + + if [ $# -eq 0 ]; then + echo "FATAL: readpid(): Missing pifile" >&2 + exit 1 + fi + + pidfile="$1" + + read -r pid <"$pidfile" + echo $pid +} + +function run() { + + if [[ -n "${RUNNER_SKIP_START}" && "${RUNNER_SKIP_START}" != "0" ]]; then + exit 0 + fi + + if [ $# -eq 0 ]; then + echo "FATAL: missing program pidfile" >&2 + elif [ $# -eq 1 ]; then + echo "FATAL: missing program to run" >&2 + help + exit 1 + fi + + pidfile="$1" + shift + + if [ -e "$pidfile" ]; then + pid=$(readpid "$pidfile") + echo "$pid" + + if pgrep --pidfile "$pidfile"; then + exit 1 + fi + fi + + "$@" 2>runner.$$.err 1>runner.$$.out 0"$pidfile" + sleep 1 + + if ! kill -0 $pid; then + echo "Process $pid does not seem to exist." >&2 + echo "The program below may not exist or may have crashed while starting:" >&2 + echo " $*" >&2 + echo " STDOUT: " >&2 + cat "runner.$$.out" >&2 + echo " STDERR: " >&2 + cat "runner.$$.err" >&2 + rm runner.$$.out runner.$$.err + exit 1 + fi + + exit 0 +} + +function stop() { + + if [ $# -eq 0 ]; then + echo "FATAL: missing signal" >&2 + exit 1 + elif [ $# -eq 1 ]; then + echo "FATAL: missing pidfile" >&2 + exit 1 + fi + + signal="$1" + pidfile="$2" + + if [ ! -e "$pidfile" ]; then + echo "FATAL: pidfile '$pidfile' does not exist" >&2 + exit 1 + fi + + if pkill "-$signal" --pidfile "$pidfile"; then + rm "$pidfile" + rm runner.$$.out runner.$$.err + exit 0 + fi + exit 1 +} + +if [ $# -eq 0 ]; then + echo "FATAL: missing arguments" >&2 + help + exit 1 +fi + +case $1 in + +start) + shift + run "$@" + ;; + +stop) + shift + stop "$@" + ;; + +help) + help + exit 0 + ;; +esac -- GitLab From 6da96341a96b374596086469bfe42850bc3a5274 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 28 Sep 2023 12:30:32 +0200 Subject: [PATCH 02/10] Change default for CARGO_BIND_PORT to 62000 --- CMakeLists.txt | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e61830..4b27c51 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,9 +141,9 @@ message(STATUS "[${PROJECT_NAME}] server bind address: ${CARGO_BIND_ADDRESS}") ### server bind port set(CARGO_BIND_PORT - "52000" + "62000" CACHE STRING - "Define the bind port for the ${PROJECT_NAME} server (default: 52000)" + "Define the bind port for the ${PROJECT_NAME} server (default: 62000)" ) message(STATUS "[${PROJECT_NAME}] server bind port: ${CARGO_BIND_PORT}") diff --git a/README.md b/README.md index f93ac25..4bf64d0 100644 --- a/README.md +++ b/README.md @@ -140,5 +140,5 @@ library (`${INSTALL_DIR}/lib/libcargo.so`) and its headers ```shell cd build/tests/ mpirun -np 4 ${INSTALL_DIR}/bin/cargo -C -./tests -S ofi+tcp://127.0.0.1:52000 +./tests -S ofi+tcp://127.0.0.1:62000 ``` -- GitLab From e0b754a1821beaa039b5dce71fdd6aa1361b10a2 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 28 Sep 2023 12:54:33 +0200 Subject: [PATCH 03/10] Add control utils for `cargo` - `cargo_ping`: Send a ping to a Cargo server - `cargo_shutdown`: Send a shutdwon request to a Cargo server --- CMakeLists.txt | 1 + src/master.cpp | 12 ++++++ src/master.hpp | 3 ++ util/CMakeLists.txt | 57 ++++++++++++++++++++++++ util/ping.cpp | 103 ++++++++++++++++++++++++++++++++++++++++++++ util/shutdown.cpp | 92 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 268 insertions(+) create mode 100644 util/CMakeLists.txt create mode 100644 util/ping.cpp create mode 100644 util/shutdown.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b27c51..536711b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -322,6 +322,7 @@ endif () add_subdirectory(etc) add_subdirectory(lib) add_subdirectory(src) +add_subdirectory(util) if(CARGO_BUILD_TESTS) add_subdirectory(tests) diff --git a/src/master.cpp b/src/master.cpp index e274eb2..18953e1 100644 --- a/src/master.cpp +++ b/src/master.cpp @@ -82,6 +82,7 @@ master_server::master_server(std::string name, std::string address, #define EXPAND(rpc_name) #rpc_name##s, &master_server::rpc_name provider::define(EXPAND(ping)); + provider::define(EXPAND(shutdown)); provider::define(EXPAND(transfer_datasets)); provider::define(EXPAND(transfer_status)); @@ -161,6 +162,17 @@ master_server::ping(const network::request& req) { req.respond(resp); } +void master_server::shutdown(const network::request& req) { + using network::get_address; + using network::rpc_info; + using proto::generic_response; + + const auto rpc = rpc_info::create(RPC_NAME(), get_address(req)); + + LOGGER_INFO("rpc {:>} body: {{}}", rpc); + server::shutdown(); +} + void master_server::transfer_datasets(const network::request& req, const std::vector& sources, diff --git a/src/master.hpp b/src/master.hpp index 77ca0fd..3d5797a 100644 --- a/src/master.hpp +++ b/src/master.hpp @@ -47,6 +47,9 @@ private: void ping(const network::request& req); + void + shutdown(const network::request& req); + void transfer_datasets(const network::request& req, const std::vector& sources, diff --git a/util/CMakeLists.txt b/util/CMakeLists.txt new file mode 100644 index 0000000..a1ee384 --- /dev/null +++ b/util/CMakeLists.txt @@ -0,0 +1,57 @@ +################################################################################ +# Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain # +# # +# This software was partially supported by the EuroHPC-funded project ADMIRE # +# (Project ID: 956748, https://www.admire-eurohpc.eu). # +# # +# This file is part of Cargo. # +# # +# Cargo is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# Cargo is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with Cargo. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +add_executable(cargo_ping) + +target_sources(cargo_ping + PRIVATE + ping.cpp +) + +target_link_libraries(cargo_ping + PUBLIC + fmt::fmt + CLI11::CLI11 + net::rpc_client + cargo +) + +add_executable(cargo_shutdown) + +target_sources(cargo_shutdown + PRIVATE + shutdown.cpp +) + +target_link_libraries(cargo_shutdown + PUBLIC + fmt::fmt + CLI11::CLI11 + net::rpc_client + cargo +) + +install(TARGETS cargo_ping cargo_shutdown + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} +) diff --git a/util/ping.cpp b/util/ping.cpp new file mode 100644 index 0000000..2449f8b --- /dev/null +++ b/util/ping.cpp @@ -0,0 +1,103 @@ +/****************************************************************************** + * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of Cargo. + * + * Cargo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Cargo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Cargo. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include + +struct ping_config { + std::string progname; + std::string server_address; +}; + +ping_config +parse_command_line(int argc, char* argv[]) { + + ping_config cfg; + + cfg.progname = std::filesystem::path{argv[0]}.filename().string(); + + CLI::App app{"Cargo ping client", cfg.progname}; + + app.add_option("-s,--server", cfg.server_address, "Server address") + ->option_text("ADDRESS") + ->required(); + + try { + app.parse(argc, argv); + return cfg; + } catch(const CLI::ParseError& ex) { + std::exit(app.exit(ex)); + } +} + +auto +parse_address(const std::string& address) { + const auto pos = address.find("://"); + if(pos == std::string::npos) { + throw std::runtime_error(fmt::format("Invalid address: {}", address)); + } + + const auto protocol = address.substr(0, pos); + return std::make_pair(protocol, address); +} + + +int +main(int argc, char* argv[]) { + + ping_config cfg = parse_command_line(argc, argv); + + try { + const auto [protocol, address] = parse_address(cfg.server_address); + network::client rpc_client{protocol}; + + if(const auto result = rpc_client.lookup(address); result.has_value()) { + const auto& endpoint = result.value(); + const auto retval = endpoint.call("ping"); + + if(retval.has_value()) { + + auto error_code = int{retval.value()}; + + fmt::print("ping RPC was successful!\n"); + fmt::print(" (server replied with: {})\n", error_code); + return EXIT_SUCCESS; + } + + fmt::print(stderr, "ping RPC failed\n"); + return EXIT_FAILURE; + + } else { + fmt::print(stderr, "Failed to lookup address: {}\n", address); + return EXIT_FAILURE; + } + } catch(const std::exception& ex) { + fmt::print(stderr, "Error: {}\n", ex.what()); + return EXIT_FAILURE; + } +} diff --git a/util/shutdown.cpp b/util/shutdown.cpp new file mode 100644 index 0000000..89b6d98 --- /dev/null +++ b/util/shutdown.cpp @@ -0,0 +1,92 @@ +/****************************************************************************** + * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of Cargo. + * + * Cargo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Cargo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Cargo. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include + +struct ping_config { + std::string progname; + std::string server_address; +}; + +ping_config +parse_command_line(int argc, char* argv[]) { + + ping_config cfg; + + cfg.progname = std::filesystem::path{argv[0]}.filename().string(); + + CLI::App app{"Cargo shutdown client", cfg.progname}; + + app.add_option("-s,--server", cfg.server_address, "Server address") + ->option_text("ADDRESS") + ->required(); + + try { + app.parse(argc, argv); + return cfg; + } catch(const CLI::ParseError& ex) { + std::exit(app.exit(ex)); + } +} + +auto +parse_address(const std::string& address) { + const auto pos = address.find("://"); + if(pos == std::string::npos) { + throw std::runtime_error(fmt::format("Invalid address: {}", address)); + } + + const auto protocol = address.substr(0, pos); + return std::make_pair(protocol, address); +} + + +int +main(int argc, char* argv[]) { + + ping_config cfg = parse_command_line(argc, argv); + + try { + const auto [protocol, address] = parse_address(cfg.server_address); + network::client rpc_client{protocol}; + + if(const auto result = rpc_client.lookup(address); result.has_value()) { + const auto& endpoint = result.value(); + endpoint.call("shutdown"); + fmt::print(stdout, "shutdown RPC sent to {}\n", address); + return EXIT_SUCCESS; + } + + fmt::print(stderr, "Failed to lookup address: {}\n", address); + return EXIT_FAILURE; + } catch(const std::exception& ex) { + fmt::print(stderr, "Error: {}\n", ex.what()); + return EXIT_FAILURE; + } +} -- GitLab From 091058130ba5e4d0ae516e70d7b1d8d7b2506731 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 28 Sep 2023 15:49:43 +0200 Subject: [PATCH 04/10] Fix tests so that they can run concurrently --- tests/CMakeLists.txt | 28 ++++++++++++++++++++++++++++ tests/tests.cpp | 18 ++++++++++-------- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 99998ab..72bb98e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -34,7 +34,35 @@ target_link_libraries( tests PUBLIC Catch2::Catch2 Boost::iostreams fmt::fmt cargo posix_file ) +# prepare the environment for the Cargo daemon +set(CARGO_TESTS_DIRECTORY "${CMAKE_BINARY_DIR}/Testing") +file(MAKE_DIRECTORY ${CARGO_TESTS_DIRECTORY}) + +set(TEST_DIRECTORY "${CARGO_TESTS_DIRECTORY}/cargo_server") +file(MAKE_DIRECTORY ${TEST_DIRECTORY}) + +set(CARGO_ADDRESS + ${CARGO_TRANSPORT_PROTOCOL}://${CARGO_BIND_ADDRESS}:${CARGO_BIND_PORT}) + +add_test(NAME start_cargo + COMMAND + ${CMAKE_SOURCE_DIR}/scripts/runner.sh start /dev/null + ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 + $ -l ${CARGO_ADDRESS} -o ${TEST_DIRECTORY}/cargo.log +) + +set_tests_properties(start_cargo + PROPERTIES FIXTURES_SETUP cargo_daemon) + +add_test(NAME stop_cargo + COMMAND cargo_shutdown --server ${CARGO_ADDRESS} +) + +set_tests_properties(stop_cargo + PROPERTIES FIXTURES_CLEANUP cargo_daemon) + catch_discover_tests( tests EXTRA_ARGS "-S ${CARGO_TRANSPORT_PROTOCOL}://${CARGO_BIND_ADDRESS}:${CARGO_BIND_PORT}" + PROPERTIES FIXTURES_REQUIRED cargo_daemon ) diff --git a/tests/tests.cpp b/tests/tests.cpp index 15b682b..6ec9d74 100644 --- a/tests/tests.cpp +++ b/tests/tests.cpp @@ -215,10 +215,11 @@ SCENARIO("Parallel reads", "[flex_stager][parallel_reads]") { cargo::server server{server_address}; - const auto sources = prepare_datasets(cargo::dataset::type::parallel, - "source-dataset-{}", NDATASETS); - const auto targets = prepare_datasets(cargo::dataset::type::posix, - "target-dataset-{}", NDATASETS); + const auto sources = + prepare_datasets(cargo::dataset::type::parallel, + "pr-source-dataset-{}", NDATASETS); + const auto targets = prepare_datasets( + cargo::dataset::type::posix, "pr-target-dataset-{}", NDATASETS); static std::vector input_files; input_files.reserve(sources.size()); @@ -275,10 +276,11 @@ SCENARIO("Parallel writes", "[flex_stager][parallel_writes]") { cargo::server server{server_address}; - const auto sources = prepare_datasets(cargo::dataset::type::posix, - "source-dataset-{}", NDATASETS); - const auto targets = prepare_datasets(cargo::dataset::type::parallel, - "target-dataset-{}", NDATASETS); + const auto sources = prepare_datasets( + cargo::dataset::type::posix, "pw-source-dataset-{}", NDATASETS); + const auto targets = + prepare_datasets(cargo::dataset::type::parallel, + "pw-target-dataset-{}", NDATASETS); static std::vector input_files; input_files.reserve(sources.size()); -- GitLab From f17f04659c376824c1291b44dd2d82fa75f74386 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 29 Sep 2023 08:55:02 +0200 Subject: [PATCH 05/10] Update .gitlab-ci.yml Add `build` stages --- .gitlab-ci.yml | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fef13ef..86059d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,34 @@ -# You can override the included template(s) by including variable overrides -# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings -# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings -# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings -# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings -# Note that environment variables can be set in several places -# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence +image: bscstorage/cargo:0.2.0-wip + stages: -- test + - build + - test + +variables: + PREFIX: + /usr/local + LD_LIBRARY_PATH: "/usr/lib/:/usr/lib64:/usr/local/lib:/usr/local/lib64" + PKG_CONFIG_PATH: "/usr/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig:/usr/lib64/openmpi/lib/pkgconfig" + +before_script: + - source /etc/profile.d/modules.sh + - module load mpi + +release: + stage: build + script: + - cmake --preset ci-release + - cmake --build builds/ci-release -j$(nproc) --target install + +debug: + stage: build + script: + - cmake --preset ci-debug + - cmake --build builds/ci-debug -j$(nproc) --target install + sast: stage: test + before_script: [] + needs: [] include: -- template: Security/SAST.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml -- GitLab From 7e8ae08b0b7db6de5ccb05fc6dd421bddaf6f8e6 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 29 Sep 2023 08:56:06 +0200 Subject: [PATCH 06/10] Add CMake presets --- CMakePresets.json | 203 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 CMakePresets.json diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000..15acccd --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,203 @@ +{ + "version": 2, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "base", + "displayName": "GCC", + "description": "Sets prefix, build, and install directories as well as common options", + "hidden": true, + "generator": "Ninja", + "binaryDir": "${sourceDir}/builds/${presetName}", + "cacheVariables": { + "CMAKE_CXX_COMPILER_LAUNCHER": "/usr/bin/ccache", + "CMAKE_C_COMPILER_LAUNCHER": "/usr/bin/ccache", + "CMAKE_PREFIX_PATH": "${sourceParentDir}/prefix", + "CMAKE_INSTALL_PREFIX": "${sourceParentDir}/prefix", + "CARGO_BUILD_TESTS": true + } + }, + { + "name": "debug", + "displayName": "Debug", + "description": "Build options for Debug", + "hidden": true, + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_CXX_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always --pedantic" + } + }, + { + "name": "release", + "displayName": "Releae", + "description": "Build options for Release", + "hidden": true, + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "gcc", + "displayName": "GCC (system default)", + "description": "Build options for GCC (system default)", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/g++", + "CMAKE_C_COMPILER": "/usr/bin/gcc", + "CMAKE_CXX_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always", + "CMAKE_C_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always" + } + }, + { + "name": "gcc-debug", + "displayName": "GCC (system default, debug)", + "description": "Build options for GCC (system default)", + "inherits": "gcc", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_C_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always -O0" + } + }, + { + "name": "gcc-10", + "displayName": "GCC 10", + "description": "Build options for GCC 10", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/g++-10", + "CMAKE_C_COMPILER": "/usr/bin/gcc-10", + "CMAKE_CXX_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always", + "CMAKE_C_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always" + } + }, + { + "name": "gcc-11", + "displayName": "GCC 11", + "description": "Build options for GCC 11", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/g++-11", + "CMAKE_C_COMPILER": "/usr/bin/gcc-11", + "CMAKE_CXX_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always", + "CMAKE_C_FLAGS": "-Wall -Wextra -Werror -fdiagnostics-color=always" + } + }, + { + "name": "clang", + "displayName": "Clang (system default)", + "description": "Build options for Clang (system default)", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/clang++", + "CMAKE_CXX_FLAGS": "-fdiagnostics-color=always", + "CMAKE_C_COMPILER": "/usr/bin/clang", + "CMAKE_C_FLAGS": "-Wno-unused-command-line-argument -fdiagnostics-color=always" + } + }, + { + "name": "clang-10", + "displayName": "Clang 10", + "description": "Build options for Clang 10", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/clang++-10", + "CMAKE_CXX_FLAGS": "-fdiagnostics-color=always", + "CMAKE_C_COMPILER": "/usr/bin/clang-10", + "CMAKE_C_FLAGS": "-Wno-unused-command-line-argument -fdiagnostics-color=always" + } + }, + { + "name": "clang-11", + "displayName": "Clang 11", + "description": "Build options for Clang 11", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/clang++-11", + "CMAKE_CXX_FLAGS": "-fdiagnostics-color=always", + "CMAKE_C_COMPILER": "/usr/bin/clang-11", + "CMAKE_C_FLAGS": "-Wno-unused-command-line-argument -fdiagnostics-color=always" + } + }, + { + "name": "clang-12", + "displayName": "Clang 12", + "description": "Build options for Clang 12", + "inherits": "base", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "/usr/bin/clang++-12", + "CMAKE_CXX_FLAGS": "-fdiagnostics-color=always", + "CMAKE_C_COMPILER": "/usr/bin/clang-12", + "CMAKE_C_FLAGS": "-Wno-unused-command-line-argument -fdiagnostics-color=always" + } + }, + { + "name": "ci", + "displayName": "CI", + "description": "Build options for CI", + "inherits": "base", + "hidden": true, + "cacheVariables": { + "CMAKE_CXX_COMPILER_LAUNCHER": null, + "CMAKE_C_COMPILER_LAUNCHER": null, + "CMAKE_C_COMPILER": "/usr/bin/gcc", + "CMAKE_CXX_COMPILER": "/usr/bin/g++", + "CMAKE_INSTALL_PREFIX": "/usr/local", + "CMAKE_PREFIX_PATH": "/usr/local", + "Boost_LIBRARY_DIR": "/usr/lib;/usr/lib64;/usr/lib64/openmpi/lib", + "CARGO_TRANSPORT_LIBRARY": "libfabric", + "CARGO_TRANSPORT_PROTOCOL": "ofi+tcp", + "CARGO_BIND_ADDRESS": "127.0.0.1", + "CARGO_BIND_PORT": "62000", + "CARGO_BUILD_TESTS": true + } + }, + { + "name": "ci-debug", + "displayName": "CI (debug)", + "description": "Build options for CI (debug)", + "inherits": ["ci", "debug"] + }, + { + "name": "ci-release", + "displayName": "CI (debug)", + "description": "Build options for CI (debug)", + "inherits": ["ci", "release"] + } + ], + "buildPresets": [ + { + "name": "core-build", + "description": "Inherits environment from base configurePreset", + "configurePreset": "base", + "hidden": true + }, + { + "name": "gcc", + "description": "Build with default GCC", + "configurePreset": "gcc", + "inherits": "core-build" + }, + { + "name": "gcc-11", + "description": "Build with GCC 11", + "configurePreset": "gcc-11", + "inherits": "core-build" + }, + { + "name": "clang", + "description": "Build with default Clang", + "configurePreset": "clang", + "inherits": "core-build" + }, + { + "name": "clang-10", + "description": "Build with Clang 10", + "configurePreset": "clang-10", + "inherits": "core-build" + } + ] +} -- GitLab From 86d68d8d4f6f360a8b43ec9bc44c20bca32fefd3 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 29 Sep 2023 10:49:19 +0200 Subject: [PATCH 07/10] Add Dockerfile for CI environment --- docker/0.2.0-wip/Dockerfile | 133 ++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 docker/0.2.0-wip/Dockerfile diff --git a/docker/0.2.0-wip/Dockerfile b/docker/0.2.0-wip/Dockerfile new file mode 100644 index 0000000..53b3b4e --- /dev/null +++ b/docker/0.2.0-wip/Dockerfile @@ -0,0 +1,133 @@ +FROM rockylinux:9.2 + +RUN set -ex \ + && yum makecache \ + && yum -y update \ + && yum -y install dnf-plugins-core \ + && yum config-manager --set-enabled crb \ + && yum -y install \ + gcc \ + gcc-c++\ + gdb \ + git \ + gnupg \ + make \ + automake \ + libtool \ + file \ + ninja-build \ + json-c-devel \ + libibverbs-devel \ + boost-devel \ + boost-openmpi-devel \ + json-c-devel \ + openmpi-devel \ + libconfig-devel \ + # install cmake 3.21+ since we need to produce JUnit XML files + && curl -OL https://github.com/Kitware/CMake/releases/download/v3.27.6/cmake-3.27.6-Linux-x86_64.sh \ + && chmod u+x ./cmake-3.27.6-Linux-x86_64.sh \ + && ./cmake-3.27.6-Linux-x86_64.sh --skip-license --prefix=/usr \ + # cleanup + && yum clean all \ + && rm -rf /var/cache/yum \ + && rm ./cmake-3.27.6-Linux-x86_64.sh + +# Download and install dependencies +RUN set -ex \ + && export LD_LIBRARY_PATH=${DEPS_INSTALL_PATH}/lib:${DEPS_INSTALL_PATH}/lib64 \ + && export PKG_CONFIG_PATH=${DEPS_INSTALL_PATH}/lib/pkgconfig:${DEPS_INSTALL_PATH}/lib64/pkgconfig \ + && cd \ + && mkdir deps \ + && cd deps \ + && git clone https://github.com/ofiwg/libfabric --recurse-submodules \ + && git clone https://github.com/pmodels/argobots --recurse-submodules \ + && git clone https://github.com/mercury-hpc/mercury --recurse-submodules \ + && git clone https://github.com/mochi-hpc/mochi-margo --recurse-submodules \ + && git clone https://github.com/USCiLab/cereal --recurse-submodules \ + && git clone https://github.com/mochi-hpc/mochi-thallium --recurse-submodules \ + \ + && cd \ + ### argobots + && cd deps/argobots \ + && ./autogen.sh \ + && mkdir build \ + && cd build \ + && CFLAGS="-ggdb3 -O0" ../configure --prefix=${DEPS_INSTALL_PATH} \ + && make install -j \ + && cd .. \ + && rm -rf build \ + && cd \ + \ + ### libfabric + && cd deps/libfabric \ + && git checkout v1.14.0rc3 \ + && ./autogen.sh \ + && mkdir build \ + && cd build \ + && CFLAGS="-ggdb3 -O0" ../configure --prefix=${DEPS_INSTALL_PATH} \ + && make install -j \ + && cd .. \ + && rm -rf build \ + && cd \ + \ + ### mercury + && cd deps/mercury \ + && mkdir build && cd build \ + && cmake \ + -DMERCURY_USE_SELF_FORWARD:BOOL=ON \ + -DBUILD_TESTING:BOOL=ON \ + -DMERCURY_USE_BOOST_PP:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DNA_USE_OFI:BOOL=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + .. \ + && make install -j \ + && cd .. \ + && rm -rf build \ + && cd \ + \ + ### mochi-margo + && cd deps/mochi-margo \ + && ./prepare.sh \ + && mkdir build \ + && cd build \ + && CFLAGS="-ggdb3 -O0" ../configure --prefix=${DEPS_INSTALL_PATH} \ + && make -j install \ + && cd .. \ + && rm -rf build \ + && cd \ + \ + ### cereal + && cd deps/cereal \ + && mkdir build \ + && cd build \ + \ + && cmake \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + -DBUILD_DOC:BOOL=OFF \ + -DBUILD_SANDBOX:BOOL=OFF \ + -DBUILD_TESTS:BOOL=OFF \ + -DSKIP_PERFORMANCE_COMPARISON:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \ + .. \ + && make -j install \ + && cd .. \ + && rm -rf build \ + && cd \ + \ + ### mochi-thallium + && cd deps/mochi-thallium \ + && mkdir build \ + && cd build \ + && cmake \ + -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + .. \ + && make -j install \ + && cd .. \ + && rm -rf build \ + && cd \ + \ + && rm -rf deps -- GitLab From c93029dd5a4a22b4e6529dceadcc51a976d1a9e5 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 29 Sep 2023 12:22:34 +0200 Subject: [PATCH 08/10] Fix issue preventing linkage in some platforms --- src/net/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt index 056dcf5..5aa4c9d 100644 --- a/src/net/CMakeLists.txt +++ b/src/net/CMakeLists.txt @@ -31,6 +31,7 @@ target_sources( ) target_link_libraries(rpc_common PUBLIC logger::logger thallium asio::asio) +set_property(TARGET rpc_common PROPERTY POSITION_INDEPENDENT_CODE ON) # get the parent directory of the current directory so we can include # headers from these libraries as `` -- GitLab From 273ca749a4c8659cc6c714f1f02fd011c6fd79fa Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 29 Sep 2023 13:05:01 +0200 Subject: [PATCH 09/10] Update .gitlab-ci.yml Add job for integration tests --- .gitlab-ci.yml | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86059d3..d3d4ebd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,8 +7,17 @@ stages: variables: PREFIX: /usr/local - LD_LIBRARY_PATH: "/usr/lib/:/usr/lib64:/usr/local/lib:/usr/local/lib64" - PKG_CONFIG_PATH: "/usr/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig:/usr/lib64/openmpi/lib/pkgconfig" + LD_LIBRARY_PATH: >- + /usr/lib/: + /usr/lib64: + /usr/local/lib: + /usr/local/lib64 + PKG_CONFIG_PATH: >- + /usr/lib/pkgconfig: + /usr/lib64/pkgconfig: + /usr/local/lib/pkgconfig: + /usr/local/lib64/pkgconfig: + /usr/lib64/openmpi/lib/pkgconfig before_script: - source /etc/profile.d/modules.sh @@ -25,6 +34,32 @@ debug: script: - cmake --preset ci-debug - cmake --build builds/ci-debug -j$(nproc) --target install + - cd builds/ci-debug + # cleanup intermediate files to save on artifact space + - grep "^rule.*\(_COMPILER_\|_STATIC_LIBRARY_\)" + $(find . -name rules.ninja) | + cut -d ' ' -f2 | + xargs -n1 ninja -t clean -r + artifacts: + expire_in: 2 days + paths: + - builds/ci-debug + +integration: + stage: test + needs: [ debug ] + variables: + OMPI_ALLOW_RUN_AS_ROOT: "1" + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: "1" + script: + - ctest --test-dir builds/ci-debug -j$(nproc) --output-junit report.xml + + artifacts: + expire_in: 1 week + paths: + - builds/ci-debug/Testing/Temporary + reports: + junit: builds/ci-debug/report.xml sast: stage: test -- GitLab From 30128df16f2dff68cdeb9c558d63918256312efc Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 29 Sep 2023 15:44:26 +0200 Subject: [PATCH 10/10] Update README.md --- README.md | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4bf64d0..1fccc2d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,14 @@ -# Cargo +
+

Cargo

+ +[![build status)](https://img.shields.io/gitlab/pipeline-status/hpc/cargo?gitlab_url=https%3A%2F%2Fstorage.bsc.es%2Fgitlab%2F&logo=gitlab)](https://storage.bsc.es/gitlab/hpc/cargo/-/pipelines) +[![latest release](https://storage.bsc.es/gitlab/hpc/cargo/-/badges/release.svg)](https://storage.bsc.es/gitlab/hpc/cargo/-/releases) +[![GitLab (self-managed)](https://img.shields.io/gitlab/license/hpc/cargo?gitlab_url=https%3A%2F%2Fstorage.bsc.es%2Fgitlab)](https://storage.bsc.es/gitlab/hpc/cargo/-/blob/main/COPYING) +[![Language](https://img.shields.io/static/v1?label=language&message=C99%20%2F%20C%2B%2B20&color=red)](https://en.wikipedia.org/wiki/C%2B%2B20) + +

A parallel data staging service for HPC clusters

+ +
Cargo is a HPC data staging service that runs alongside applications helping them to transfer data in parallel between local and shared storage tiers. @@ -137,8 +147,26 @@ library (`${INSTALL_DIR}/lib/libcargo.so`) and its headers ## Testing +Tests can be run automatically with CTest: + +```shell +cd build +ctest -VV --output-on-failure --stop-on-failure -j 8 +``` + +When this happens, a Cargo server with 3 workers is automatically started +(via `mpirun`/`mpiexec`) and stopped (via RPC) so that tests can progress. + +Alternatively, during development one may desire to run the Cargo server +manually and then the tests. In this case, the following commands can be used: + ```shell -cd build/tests/ -mpirun -np 4 ${INSTALL_DIR}/bin/cargo -C -./tests -S ofi+tcp://127.0.0.1:62000 +# start the Cargo server with 3 workers. The server will be listening on +# port 62000 and will communicate with workers via MPI messages. The server can +# be stopped with Ctrl+C, `kill -TERM ` or `cargo_shutdown
`.) +mpirun -np 4 ${INSTALL_DIR}/bin/cargo -l ofi+tcp://127.0.0.1:62000 + +# run the tests +cd build +RUNNER_SKIP_START=1 ctest -VV --output-on-failure --stop-on-failure -j 8 ``` -- GitLab