From 38e21b1c73a2c261abf295791448921621650b44 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 30 Aug 2023 17:39:04 +0200 Subject: [PATCH 1/6] Update `logger` module --- lib/CMakeLists.txt | 2 +- src/CMakeLists.txt | 2 - src/logger/CMakeLists.txt | 14 +- src/logger/logger.cpp | 221 +++++++++++++++++++ src/logger/logger.h | 79 +++++++ src/logger/logger.hpp | 360 +++++++++++++++++++++++++++++++ src/logger/logger/CMakeLists.txt | 29 --- src/logger/logger/logger.hpp | 346 ----------------------------- src/logger/macros.h | 155 +++++++++++++ src/worker.cpp | 3 +- 10 files changed, 829 insertions(+), 382 deletions(-) create mode 100644 src/logger/logger.cpp create mode 100644 src/logger/logger.h create mode 100644 src/logger/logger.hpp delete mode 100644 src/logger/logger/CMakeLists.txt delete mode 100644 src/logger/logger/logger.hpp create mode 100644 src/logger/macros.h diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 5717048..3e42647 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -36,7 +36,7 @@ target_include_directories( set_target_properties(cargo PROPERTIES PUBLIC_HEADER "${public_headers}") -target_link_libraries(cargo PRIVATE logger fmt::fmt thallium) +target_link_libraries(cargo PRIVATE logger::logger fmt::fmt thallium) ## Install library + targets ################################################### diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c4b5f9c..2f34620 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -23,7 +23,6 @@ ################################################################################ add_subdirectory(utils) -add_subdirectory(config) add_subdirectory(logger) add_subdirectory(net) add_subdirectory(posix_file) @@ -52,7 +51,6 @@ target_include_directories( target_link_libraries( cargo_server PRIVATE config - logger rpc_server cargo fmt::fmt diff --git a/src/logger/CMakeLists.txt b/src/logger/CMakeLists.txt index f268735..4be99e3 100644 --- a/src/logger/CMakeLists.txt +++ b/src/logger/CMakeLists.txt @@ -22,10 +22,18 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ -add_subdirectory(logger) +# get the parent directory of the current directory so we can include +# headers from these libraries as `` +get_filename_component(PARENT_DIRECTORY "../" ABSOLUTE) -target_include_directories( - logger INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} +add_library(logger STATIC) +target_sources( + logger + INTERFACE logger.h logger.hpp macros.h + PRIVATE logger.cpp ) +target_include_directories(logger PUBLIC ${PARENT_DIRECTORY}) +target_link_libraries(logger PUBLIC spdlog::spdlog fmt::fmt) set_property(TARGET logger PROPERTY POSITION_INDEPENDENT_CODE ON) +add_library(logger::logger ALIAS logger) diff --git a/src/logger/logger.cpp b/src/logger/logger.cpp new file mode 100644 index 0000000..872d1f5 --- /dev/null +++ b/src/logger/logger.cpp @@ -0,0 +1,221 @@ +/****************************************************************************** + * Copyright 2021-2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of Cargo. + * + * Cargo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Cargo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Cargo. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "logger.hpp" +#include "logger.h" + +//////////////////////////////////////////////////////////////////////////////// +// Function implementations for the C API +//////////////////////////////////////////////////////////////////////////////// + +void +logger_setup(const char* ident, logger_type type, const char* log_file) { + + constexpr auto get_cxx_type = [](logger_type t) { + switch(t) { + case CONSOLE_LOGGER: + return logger::logger_type::console; + case CONSOLE_COLOR_LOGGER: + return logger::logger_type::console_color; + case FILE_LOGGER: + return logger::logger_type::file; + case SYSLOG_LOGGER: + return logger::logger_type::syslog; + default: + return logger::logger_type::console; + } + }; + + logger::create_default_logger( + logger::logger_config{ident, get_cxx_type(type), log_file}); +} + +void +logger_log(enum logger_level level, const char* fmt, ...) { + + using logger::logger_base; + + if(const auto logger = logger_base::get_default_logger(); logger) { + + std::array msg; // NOLINT + va_list args; + va_start(args, fmt); + vsnprintf(msg.data(), msg.size(), fmt, args); + va_end(args); + + switch(level) { + case info: + logger->info(msg.data()); + break; + case debug: + logger->debug(msg.data()); + break; + case warn: + logger->warn(msg.data()); + break; + case error: + logger->error(msg.data()); + break; + case critical: + logger->critical(msg.data()); + break; + } + } +} + +void +logger_destroy() { + using logger::logger_base; + + if(logger_base::get_default_logger()) { + ::logger::destroy_default_logger(); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Function implementations for the C++ API +//////////////////////////////////////////////////////////////////////////////// + +namespace { + +/** + * @brief Creates a logger of the given type. + * + * @tparam Logger Type of the logger to create. + * @param config Configuration for the logger. + * @return std::shared_ptr Pointer to the created logger. + */ +template +std::shared_ptr +create_logger(const logger::logger_config& config) + requires(std::is_same_v || + std::is_same_v) { + + const auto create_helper = [&config]() { + switch(config.type()) { + case logger::console: { + if constexpr(std::is_same_v) { + return spdlog::stdout_logger_st(config.ident()); + } + return spdlog::stdout_logger_mt( + config.ident()); + } + case logger::console_color: + if constexpr(std::is_same_v) { + return spdlog::stdout_color_st(config.ident()); + } + return spdlog::stdout_color_mt( + config.ident()); + case logger::file: + if constexpr(std::is_same_v) { + return spdlog::basic_logger_st( + config.ident(), config.log_file().value_or(""), + true); + } + return spdlog::basic_logger_mt( + config.ident(), config.log_file().value_or(""), true); + case logger::syslog: + if constexpr(std::is_same_v) { + return spdlog::syslog_logger_st("syslog", config.ident(), + LOG_PID); + } + return spdlog::syslog_logger_mt("syslog", config.ident(), + LOG_PID); + default: + throw std::invalid_argument("Unknown logger type"); + } + }; + + try { + auto logger = create_helper(); + assert(logger != nullptr); + logger->set_pattern(logger::default_pattern); + +#ifdef LOGGER_ENABLE_DEBUG + logger->set_level(spdlog::level::debug); +#endif + spdlog::drop_all(); + return logger; + } catch(const spdlog::spdlog_ex& ex) { + throw std::runtime_error("logger initialization failed: " + + std::string(ex.what())); + } +} + +} // namespace + +namespace logger { + +logger_base::logger_base(logger::logger_config config) + : m_config(std::move(config)), + m_internal_logger(::create_logger(m_config)) {} + +const logger_config& +logger_base::config() const { + return m_config; +} + +std::shared_ptr& +logger_base::get_default_logger() { + static std::shared_ptr s_global_logger; + return s_global_logger; +} + +void +logger_base::enable_debug() const { + m_internal_logger->set_level(spdlog::level::debug); +} + +void +logger_base::flush() { + m_internal_logger->flush(); +} + +async_logger::async_logger(const logger_config& config) : logger_base(config) { + try { + m_internal_logger = ::create_logger(config); + } catch(const spdlog::spdlog_ex& ex) { + throw std::runtime_error("logger initialization failed: " + + std::string(ex.what())); + } +} + +sync_logger::sync_logger(const logger_config& config) : logger_base(config) { + try { + m_internal_logger = ::create_logger(config); + } catch(const spdlog::spdlog_ex& ex) { + throw std::runtime_error("logger initialization failed: " + + std::string(ex.what())); + } +} + +} // namespace logger diff --git a/src/logger/logger.h b/src/logger/logger.h new file mode 100644 index 0000000..64c9c37 --- /dev/null +++ b/src/logger/logger.h @@ -0,0 +1,79 @@ +/****************************************************************************** + * Copyright 2021-2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of Cargo. + * + * Cargo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Cargo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Cargo. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#ifndef SCORD_LOGGER_H +#define SCORD_LOGGER_H + +#include "macros.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOGGER_MSG_MAX_LEN 2048 + +enum logger_type { + CONSOLE_LOGGER, + CONSOLE_COLOR_LOGGER, + FILE_LOGGER, + SYSLOG_LOGGER +}; +enum logger_level { info, debug, warn, error, critical }; + +/** + * Initialize a global logger. + * + * Valid logger types: + * - console + * - console color + * - file + * + * @param ident The identifier that should be used when emitting messages + * @param type The type of logger desired. + * @param log_file The path to the log file where messages should be written. + */ +void +setup_logger(const char* ident, enum logger_type type, const char* log_file); + +/** + * Emit a message. + * + * @param level The level for the message. + * @param fmt A vprintf()-compatible format string. + * @param ... A sequence of arguments corresponding to the format string. + */ +void +logger_log(enum logger_level level, const char* fmt, ...); + +/** + * Destroy the global logger. + */ +void +destroy_logger(); + +#ifdef __cplusplus +}; // extern "C" +#endif + +#endif // SCORD_LOGGER_H diff --git a/src/logger/logger.hpp b/src/logger/logger.hpp new file mode 100644 index 0000000..891be9c --- /dev/null +++ b/src/logger/logger.hpp @@ -0,0 +1,360 @@ +/****************************************************************************** + * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of Cargo. + * + * Cargo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Cargo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Cargo. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#ifndef LOGGER_HPP +#define LOGGER_HPP + +#include +#include +#include +#include +#include + +#include "macros.h" + +#if FMT_VERSION < 50000 +namespace fmt { +template +inline const void* +ptr(const T* p) { + return p; +} +} // namespace fmt +#endif // FMT_VERSION + +namespace logger { + +enum logger_type { + console, + console_color, + file, + syslog, +}; + +class logger_config { + +public: + logger_config() = default; + + explicit logger_config(std::string ident, logger_type type, + std::optional log_file = {}) + : m_ident(std::move(ident)), m_type(type), + m_log_file(std::move(log_file)) {} + + const std::string& + ident() const { + return m_ident; + } + + logger_type + type() const { + return m_type; + } + + const std::optional& + log_file() const { + return m_log_file; + } + +private: + std::string m_ident; + logger_type m_type = console_color; + std::optional m_log_file; +}; + + +/** + * @brief The default log pattern + * + * This is the default log pattern used by the logger. + * It can be used to create new loggers with the same + * configuration. + * + * The default log pattern is: + * + * @code + * %^[%Y-%m-%d %T.%f] [%n] [%t] [%l]%$ %v + * @endcode + * + * The output of the default log pattern is: + * + * @code + * [2021-01-01 00:00:00.000000] [prog] [12345] [info] Message + * @endcode + * + * Where: + * - 2021-01-01 00:00:00.000000 is the current date and time + * - prog is the name of the logger + * - 12345 is the thread id + * - info is the log level + * - Message is the log message + * + * The following format specifiers are available: + * %Y - Year in 4 digits + * %m - month 1-12 + * %d - day 1-31 + * %T - ISO 8601 time format (HH:MM:SS) + * %f - microsecond part of the current second + * %E - epoch (microseconds precision) + * %n - logger's name + * %t - thread id + * %l - log level + * %v - message + */ +static constexpr auto default_pattern = + "%^[%Y-%m-%d %T.%f] [%n] [%t] [%l]%$ %v"; + +/** + * @brief The logger_base class + * + * This class is a wrapper around spdlog::logger, and provides common + * functionality to all the logger implementations. It also provides a + * default logger that can be used by the rest of the code by using the + * static member function get_default_logger(). + * + * @note This class should not be used directly. It is intended to serve as a + * base class for the different logger implementations. + */ +class logger_base { + +protected: + logger_base() = default; + explicit logger_base(logger_config config); + +public: + logger_base(const logger_base& /*rhs*/) = delete; + logger_base& + operator=(const logger_base& /*rhs*/) = delete; + +protected: + logger_base(logger_base&& /*other*/) = default; + logger_base& + operator=(logger_base&& /*other*/) = default; + ~logger_base() = default; + +public: + const logger_config& + config() const; + + static std::shared_ptr& + get_default_logger(); + + void + enable_debug() const; + + void + flush(); + + template + inline void + info(fmt::format_string fmt, Args&&... args) { + m_internal_logger->info(fmt, std::forward(args)...); + } + + template + inline void + debug(fmt::format_string fmt, Args&&... args) { + m_internal_logger->debug(fmt, std::forward(args)...); + } + + template + inline void + warn(fmt::format_string fmt, Args&&... args) { + m_internal_logger->warn(fmt, std::forward(args)...); + } + + template + inline void + error(fmt::format_string fmt, Args&&... args) { + m_internal_logger->error(fmt, std::forward(args)...); + } + + [[maybe_unused]] static inline std::string + errno_message(int errno_value) { + // 1024 should be more than enough for most locales + constexpr const std::size_t MAX_ERROR_MSG = 1024; + std::array errstr{}; + char* msg = strerror_r(errno_value, errstr.data(), MAX_ERROR_MSG); + return std::string{msg}; + } + + template + inline void + error_errno(fmt::format_string fmt, Args&&... args) { + + const int saved_errno = errno; + + constexpr const std::size_t MAX_ERROR_MSG = 128; + std::array errstr; // NOLINT + + fmt::string_view sv{fmt}; + std::string str_fmt{sv.begin(), sv.end()}; + str_fmt += ": {}"; + + char* msg = strerror_r(saved_errno, errstr.data(), MAX_ERROR_MSG); + + m_internal_logger->error(fmt::runtime(str_fmt), + std::forward(args)..., msg); + } + + template + inline void + critical(fmt::format_string fmt, Args&&... args) { + m_internal_logger->critical(fmt, std::forward(args)...); + } + + template + inline void + info(const T& msg) { + m_internal_logger->info(msg); + } + + template + inline void + debug(const T& msg) { + m_internal_logger->debug(msg); + } + + template + inline void + warn(const T& msg) { + m_internal_logger->warn(msg); + } + + template + inline void + error(const T& msg) { + m_internal_logger->error(msg); + } + + template + inline void + critical(const T& msg) { + m_internal_logger->critical(msg); + } + + template + [[deprecated]] [[maybe_unused]] static inline std::string + build_message(Args&&... args) { + + // see: + // 1. + // https://stackoverflow.com/questions/27375089/what-is-the-easiest-way-to-print-a-variadic-parameter-pack-using-stdostream + // 2. + // https://stackoverflow.com/questions/25680461/variadic-template-pack-expansion/25683817#25683817 + + std::stringstream ss; + + using expander = int[]; + (void) expander{0, (void(ss << std::forward(args)), 0)...}; + + return ss.str(); + } + +protected: + logger_config m_config; + std::shared_ptr m_internal_logger; +}; + +/** + * @brief Synchronous logger implementation + * + * This class is a wrapper around spdlog::logger. It provides + * a synchronous interface to the spdlog logger. + */ +class sync_logger : public logger_base { +public: + explicit sync_logger(const logger_config& config); +}; + +/** + * @brief Asynchronous logger implementation + * + * This class is a wrapper around spdlog::async_logger. It + * provides an asynchronous interface to the spdlog logger. + */ +class async_logger : public logger_base { +public: + explicit async_logger(const logger_config& config); +}; + +// the following static functions can be used to interact +// with a globally registered async logger instance + +/** + * @brief Create a default logger instance + * + * @tparam Args variadic template parameter pack for the logger constructor + * arguments + * @param args arguments for the logger constructor + */ +template +static inline void +create_default_logger(Args&&... args) { + async_logger::get_default_logger() = + std::make_shared(logger_config{args...}); +} + +/** + * @brief Register an existing logger instance as the default logger + * + * @param config logger configuration + */ +[[maybe_unused]] static inline void +set_default_logger(async_logger&& lg) { + async_logger::get_default_logger() = + std::make_shared(std::move(lg)); +} + +/** + * @brief Destroy the default logger instance + */ +[[maybe_unused]] static inline void +destroy_default_logger() { + async_logger::get_default_logger().reset(); +} + +/** + * @brief Get the default logger instance + * + * @return A shared pointer to the default logger instance + */ +[[maybe_unused]] static inline auto +get_default_logger() { + return async_logger::get_default_logger(); +} + +/** + * @brief Flush the default logger instance + */ +[[maybe_unused]] static inline void +flush_default_logger() { + if(auto lg = async_logger::get_default_logger(); lg) { + lg->flush(); + } +} + +} // namespace logger + +#endif /* LOGGER_HPP */ diff --git a/src/logger/logger/CMakeLists.txt b/src/logger/logger/CMakeLists.txt deleted file mode 100644 index 4b8f089..0000000 --- a/src/logger/logger/CMakeLists.txt +++ /dev/null @@ -1,29 +0,0 @@ -################################################################################ -# Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain # -# # -# This software was partially supported by the EuroHPC-funded project ADMIRE # -# (Project ID: 956748, https://www.admire-eurohpc.eu). # -# # -# This file is part of scord. # -# # -# scord is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# scord is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with scord. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ - -add_library(logger INTERFACE) - -target_sources(logger INTERFACE logger.hpp) - -target_link_libraries(logger INTERFACE spdlog::spdlog fmt::fmt) diff --git a/src/logger/logger/logger.hpp b/src/logger/logger/logger.hpp deleted file mode 100644 index 637c618..0000000 --- a/src/logger/logger/logger.hpp +++ /dev/null @@ -1,346 +0,0 @@ -/****************************************************************************** - * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of Cargo. - * - * Cargo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Cargo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Cargo. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#ifndef LOGGER_HPP -#define LOGGER_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -#if FMT_VERSION < 50000 -namespace fmt { -template -inline const void* -ptr(const T* p) { - return p; -} -} // namespace fmt -#endif // FMT_VERSION - -namespace fs = std::filesystem; - -class logger { - -public: - logger(const std::string& ident, const std::string& type, - const fs::path& log_file = "") { - - try { - - if(type == "console") { - m_internal_logger = - spdlog::stdout_logger_mt(ident); - } else if(type == "console color") { - m_internal_logger = - spdlog::stdout_color_mt(ident); - } else if(type == "file") { - m_internal_logger = - spdlog::basic_logger_mt( - ident, log_file.string(), true); - - } -#ifdef SPDLOG_ENABLE_SYSLOG - else if(type == "syslog") { - m_internal_logger = - spdlog::syslog_logger("syslog", ident, LOG_PID); - } -#endif - else { - throw std::invalid_argument("Unknown logger type: '" + type + - "'"); - } - - assert(m_internal_logger != nullptr); - - // %Y - Year in 4 digits - // %m - month 1-12 - // %d - day 1-31 - // %T - ISO 8601 time format (HH:MM:SS) - // %f - microsecond part of the current second - // %E - epoch (microseconds precision) - // %n - logger's name - // %t - thread id - // %l - log level - // %v - message - // m_internal_logger->set_pattern("[%Y-%m-%d %T.%f] [%E] [%n] [%t] - // [%l] %v"); - m_internal_logger->set_pattern( - "%^[%Y-%m-%d %T.%f] [%n] [%t] [%l]%$ %v"); - -#ifdef __LOGGER_ENABLE_DEBUG__ - enable_debug(); -#endif - - spdlog::drop_all(); - - // globally register the logger so that it can be accessed - // using spdlog::get(logger_name) - // spdlog::register_logger(m_internal_logger); - } catch(const spdlog::spdlog_ex& ex) { - throw std::runtime_error("logger initialization failed: " + - std::string(ex.what())); - } - } - - logger(const logger& /*rhs*/) = delete; - logger& - operator=(const logger& /*rhs*/) = delete; - logger(logger&& /*other*/) = default; - logger& - operator=(logger&& /*other*/) = default; - - ~logger() { - spdlog::drop_all(); - } - - // the following static functions can be used to interact - // with a globally registered logger instance - - template - static inline void - create_global_logger(Args&&... args) { - global_logger() = std::make_shared(args...); - } - - static inline void - register_global_logger(logger&& lg) { - global_logger() = std::make_shared(std::move(lg)); - } - - static inline std::shared_ptr& - get_global_logger() { - return global_logger(); - } - - static inline void - destroy_global_logger() { - global_logger().reset(); - } - - // the following member functions can be used to interact - // with a specific logger instance - inline void - enable_debug() const { - m_internal_logger->set_level(spdlog::level::debug); - } - - inline void - flush() { - m_internal_logger->flush(); - } - - template - inline void - info(fmt::format_string fmt, Args&&... args) { - m_internal_logger->info(fmt, std::forward(args)...); - } - - template - inline void - debug(fmt::format_string fmt, Args&&... args) { - m_internal_logger->debug(fmt, std::forward(args)...); - } - - template - inline void - warn(fmt::format_string fmt, Args&&... args) { - m_internal_logger->warn(fmt, std::forward(args)...); - } - - template - inline void - error(fmt::format_string fmt, Args&&... args) { - m_internal_logger->error(fmt, std::forward(args)...); - } - - static inline std::string - errno_message(int errno_value) { - // 1024 should be more than enough for most locales - constexpr const std::size_t MAX_ERROR_MSG = 1024; - std::array errstr{}; - char* msg = strerror_r(errno_value, errstr.data(), MAX_ERROR_MSG); - return std::string{msg}; - } - - template - inline void - error_errno(fmt::format_string fmt, Args&&... args) { - - const int saved_errno = errno; - - constexpr const std::size_t MAX_ERROR_MSG = 128; - std::array errstr; // NOLINT - - fmt::string_view sv{fmt}; - std::string str_fmt{sv.begin(), sv.end()}; - str_fmt += ": {}"; - - char* msg = strerror_r(saved_errno, errstr.data(), MAX_ERROR_MSG); - - m_internal_logger->error(fmt::runtime(str_fmt), - std::forward(args)..., msg); - } - - template - inline void - critical(fmt::format_string fmt, Args&&... args) { - m_internal_logger->critical(fmt, std::forward(args)...); - } - - template - inline void - info(const T& msg) { - m_internal_logger->info(msg); - } - - template - inline void - debug(const T& msg) { - m_internal_logger->debug(msg); - } - - template - inline void - warn(const T& msg) { - m_internal_logger->warn(msg); - } - - template - inline void - error(const T& msg) { - m_internal_logger->error(msg); - } - - template - inline void - critical(const T& msg) { - m_internal_logger->critical(msg); - } - - template - static inline std::string - build_message(Args&&... args) { - - // see: - // 1. - // https://stackoverflow.com/questions/27375089/what-is-the-easiest-way-to-print-a-variadic-parameter-pack-using-stdostream - // 2. - // https://stackoverflow.com/questions/25680461/variadic-template-pack-expansion/25683817#25683817 - - std::stringstream ss; - - using expander = int[]; - (void) expander{0, (void(ss << std::forward(args)), 0)...}; - - return ss.str(); - } - -private: - static std::shared_ptr& - global_logger() { - static std::shared_ptr s_global_logger; - return s_global_logger; - } - -private: - std::shared_ptr m_internal_logger; - std::string m_type; -}; - -// some macros to make it more convenient to use the global logger -#define LOGGER_INIT(...) \ - do { \ - logger::create_global_logger(__VA_ARGS__); \ - } while(0); - -#define LOGGER_INFO(...) \ - do { \ - if(logger::get_global_logger()) { \ - logger::get_global_logger()->info(__VA_ARGS__); \ - } \ - } while(0); - - -#ifdef __LOGGER_ENABLE_DEBUG__ - -#define LOGGER_DEBUG(...) \ - do { \ - if(logger::get_global_logger()) { \ - logger::get_global_logger()->debug(__VA_ARGS__); \ - } \ - } while(0); - -#define LOGGER_FLUSH() \ - do { \ - if(logger::get_global_logger()) { \ - logger::get_global_logger()->flush(); \ - } \ - } while(0); - -#else - -#define LOGGER_DEBUG(...) \ - do { \ - } while(0); -#define LOGGER_FLUSH() \ - do { \ - } while(0); - -#endif // __LOGGER_ENABLE_DEBUG__ - -#define LOGGER_WARN(...) \ - do { \ - if(logger::get_global_logger()) { \ - logger::get_global_logger()->warn(__VA_ARGS__); \ - } \ - } while(0); - -#define LOGGER_ERROR(...) \ - do { \ - if(logger::get_global_logger()) { \ - logger::get_global_logger()->error(__VA_ARGS__); \ - } \ - } while(0); - -#define LOGGER_ERRNO(...) \ - do { \ - if(logger::get_global_logger()) { \ - logger::get_global_logger()->error_errno(__VA_ARGS__); \ - } \ - } while(0); - -#define LOGGER_CRITICAL(...) \ - do { \ - if(logger::get_global_logger()) { \ - logger::get_global_logger()->critical(__VA_ARGS__); \ - } \ - } while(0); - -#endif /* LOGGER_HPP */ diff --git a/src/logger/macros.h b/src/logger/macros.h new file mode 100644 index 0000000..bcb17e6 --- /dev/null +++ b/src/logger/macros.h @@ -0,0 +1,155 @@ +/****************************************************************************** + * Copyright 2021-2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of Cargo. + * + * Cargo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Cargo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Cargo. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + + +#ifndef LOGGER_MACROS_H +#define LOGGER_MACROS_H + +/* logger macros for C++ code */ +#ifdef __cplusplus + +#define LOGGER_INIT(...) \ + do { \ + logger::create_default_logger(__VA_ARGS__); \ + } while(0); + +#define LOGGER_INFO(...) \ + do { \ + if(logger::get_default_logger()) { \ + logger::get_default_logger()->info(__VA_ARGS__); \ + } \ + } while(0); + + +#ifdef LOGGER_ENABLE_DEBUG + +#define LOGGER_DEBUG(...) \ + do { \ + if(logger::get_default_logger()) { \ + logger::get_default_logger()->debug(__VA_ARGS__); \ + } \ + } while(0); + +#define LOGGER_FLUSH() \ + do { \ + if(logger::get_default_logger()) { \ + logger::get_default_logger()->flush(); \ + } \ + } while(0); + +#else // ! LOGGER_ENABLE_DEBUG + +#define LOGGER_DEBUG(...) \ + do { \ + } while(0); +#define LOGGER_FLUSH() \ + do { \ + } while(0); + +#endif // LOGGER_ENABLE_DEBUG + +#define LOGGER_WARN(...) \ + do { \ + if(logger::get_default_logger()) { \ + logger::get_default_logger()->warn(__VA_ARGS__); \ + } \ + } while(0); + +#define LOGGER_ERROR(...) \ + do { \ + if(logger::get_default_logger()) { \ + logger::get_default_logger()->error(__VA_ARGS__); \ + } \ + } while(0); + +#define LOGGER_ERRNO(...) \ + do { \ + if(logger::get_default_logger()) { \ + logger::get_default_logger()->error_errno(__VA_ARGS__); \ + } \ + } while(0); + +#define LOGGER_CRITICAL(...) \ + do { \ + if(logger::get_default_logger()) { \ + logger::get_default_logger()->critical(__VA_ARGS__); \ + } \ + } while(0); + +#else // ! __cplusplus + +/* logger macros for C code */ + +// clang-format off +#define ARGSN(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, N, ...) N +#define RSEQN() 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 +// clang-format on +#define EXPAND(x) x +#define COUNT_ARGS(...) EXPAND(COUNT_ARGS_HELPER(__VA_ARGS__, RSEQN())) +#define COUNT_ARGS_HELPER(...) EXPAND(ARGSN(__VA_ARGS__)) +#define GET_NAME_HELPER(name, n) name##n +#define GET_NAME(name, n) GET_NAME_HELPER(name, n) +#define GET_MACRO(func, ...) \ + GET_NAME(func, EXPAND(COUNT_ARGS(__VA_ARGS__)))(__VA_ARGS__) + +#define LOGGER_LOG(level, ...) GET_MACRO(LOGGER_LOG, level, __VA_ARGS__) +#define LOGGER_LOG2(level, msg) logger_log(level, "%s", msg) +#define LOGGER_LOG3 LOGGER_LOGN +#define LOGGER_LOG4 LOGGER_LOGN +#define LOGGER_LOG5 LOGGER_LOGN +#define LOGGER_LOG6 LOGGER_LOGN +#define LOGGER_LOG7 LOGGER_LOGN +#define LOGGER_LOG8 LOGGER_LOGN +#define LOGGER_LOG9 LOGGER_LOGN +#define LOGGER_LOG10 LOGGER_LOGN +#define LOGGER_LOG11 LOGGER_LOGN +#define LOGGER_LOG12 LOGGER_LOGN +#define LOGGER_LOG13 LOGGER_LOGN +#define LOGGER_LOG14 LOGGER_LOGN +#define LOGGER_LOG15 LOGGER_LOGN +#define LOGGER_LOG16 LOGGER_LOGN +#define LOGGER_LOGN(level, fmt, ...) logger_log(level, fmt, __VA_ARGS__) + +#define LOGGER_INFO(fmt, ...) LOGGER_LOG(info, fmt, ##__VA_ARGS__); + +#ifdef LOGGER_ENABLE_DEBUG +#define LOGGER_DEBUG(fmt, ...) LOGGER_LOG(debug, fmt, ##__VA_ARGS__); +#endif + +#define LOGGER_WARN(fmt, ...) LOGGER_LOG(warn, fmt, ##__VA_ARGS__); + +#define LOGGER_ERROR(fmt, ...) LOGGER_LOG(error, fmt, ##__VA_ARGS__); + +#endif + +#define LOGGER_EVAL(expr, L1, L2, ...) \ + do { \ + if(expr) { \ + LOGGER_##L1(__VA_ARGS__); \ + } else { \ + LOGGER_##L2(__VA_ARGS__); \ + } \ + } while(0); + +#endif // LOGGER_MACROS_H diff --git a/src/worker.cpp b/src/worker.cpp index 23733cf..5772198 100644 --- a/src/worker.cpp +++ b/src/worker.cpp @@ -310,7 +310,8 @@ worker() { ranks_to_exclude.end()), 0); - LOGGER_INIT(fmt::format("worker_{:03}", world.rank()), "console color"); + LOGGER_INIT(fmt::format("worker_{:03}", world.rank()), + logger::console_color); // Initialization finished LOGGER_INFO("Staging process initialized (world_rank {}, workers_rank: {})", -- GitLab From 8d8f3f85c73d9cf49829932578a36d3eb22f767e Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 31 Aug 2023 09:51:01 +0200 Subject: [PATCH 2/6] Update `network` module --- CMakeLists.txt | 35 +++++- src/master.cpp | 4 +- src/net/CMakeLists.txt | 42 ++++++- src/net/client.cpp | 59 ++++++++++ src/net/client.hpp | 51 ++++++++ src/net/endpoint.cpp | 40 +++++++ src/net/endpoint.hpp | 91 +++++++++++++++ src/net/net/CMakeLists.txt | 33 ------ src/net/request.hpp | 43 +++++++ src/net/serialization.hpp | 70 +++++++++++ src/net/{net => }/server.cpp | 219 +++++++++++++++++++---------------- src/net/{net => }/server.hpp | 82 +++++++------ src/net/signal_listener.hpp | 121 +++++++++++++++++++ src/net/utilities.hpp | 136 ++++++++++++++++++++++ src/worker.cpp | 1 + 15 files changed, 847 insertions(+), 180 deletions(-) create mode 100644 src/net/client.cpp create mode 100644 src/net/client.hpp create mode 100644 src/net/endpoint.cpp create mode 100644 src/net/endpoint.hpp delete mode 100644 src/net/net/CMakeLists.txt create mode 100644 src/net/request.hpp create mode 100644 src/net/serialization.hpp rename src/net/{net => }/server.cpp (63%) rename src/net/{net => }/server.hpp (60%) create mode 100644 src/net/signal_listener.hpp create mode 100644 src/net/utilities.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b58da3..f1ac6da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,20 +4,20 @@ # This software was partially supported by the EuroHPC-funded project ADMIRE # # (Project ID: 956748, https://www.admire-eurohpc.eu). # # # -# This file is part of cargo. # +# This file is part of Cargo. # # # -# cargo is free software: you can redistribute it and/or modify # +# Cargo is free software: you can redistribute it and/or modify # # it under the terms of the GNU General Public License as published by # # the Free Software Foundation, either version 3 of the License, or # # (at your option) any later version. # # # -# cargo is distributed in the hope that it will be useful, # +# Cargo is distributed in the hope that it will be useful, # # but WITHOUT ANY WARRANTY; without even the implied warranty of # # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # # GNU General Public License for more details. # # # # You should have received a copy of the GNU General Public License # -# along with cargo. If not, see . # +# along with Cargo. If not, see . # # # # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ @@ -258,6 +258,33 @@ FetchContent_Declare( FetchContent_MakeAvailable(expected) +### Threads: required by ASIO +find_package(Threads REQUIRED) + +### ASIO: used for signal handling +### +### ASIO is based on autotools and not CMake package. We can use +### FetchContent to download it but we need to manually define the imported +### target ourselves rather than relying on FetchContent_MakeAvailable() +### to do it. +message(STATUS "[${PROJECT_NAME}] Searching for ASIO") +FetchContent_Declare( + asio + GIT_REPOSITORY https://github.com/chriskohlhoff/asio.git + GIT_TAG asio-1-18-2 + CONFIGURE_COMMAND "" BUILD_COMMAND "" +) +FetchContent_GetProperties(asio) +if(NOT asio_POPULATED) + FetchContent_Populate(asio) +endif() + +add_library(asio INTERFACE) +target_include_directories(asio INTERFACE ${asio_SOURCE_DIR}/asio/include) +target_link_libraries(asio INTERFACE Threads::Threads) +add_library(asio::asio ALIAS asio) + + pkg_check_modules(LIBCONFIG IMPORTED_TARGET libconfig>=1.4.9) if (CARGO_BUILD_TESTS) diff --git a/src/master.cpp b/src/master.cpp index 38240f4..cb5eb0e 100644 --- a/src/master.cpp +++ b/src/master.cpp @@ -94,7 +94,7 @@ ping(const thallium::request& req) { } void -transfer_datasets(const net::request& req, +transfer_datasets(const network::request& req, const std::vector& sources, const std::vector& targets) { @@ -138,7 +138,7 @@ transfer_datasets(const net::request& req, void master(const config::settings& cfg) { - net::server daemon(cfg); + network::server daemon(cfg); daemon.set_handler("ping"s, handlers::ping); daemon.set_handler("transfer_datasets"s, handlers::transfer_datasets); diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt index e906060..056dcf5 100644 --- a/src/net/CMakeLists.txt +++ b/src/net/CMakeLists.txt @@ -22,10 +22,46 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ -add_subdirectory(net) +add_library(rpc_common OBJECT) +target_sources( + rpc_common + INTERFACE endpoint.hpp request.hpp serialization.hpp utilities.hpp + signal_listener.hpp + PRIVATE endpoint.cpp +) + +target_link_libraries(rpc_common PUBLIC logger::logger thallium asio::asio) -target_include_directories( - rpc_server INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} +# get the parent directory of the current directory so we can include +# headers from these libraries as `` +get_filename_component(PARENT_DIRECTORY "../" ABSOLUTE) + +add_library(rpc_client STATIC) +target_sources( + rpc_client + INTERFACE client.hpp + PRIVATE client.cpp ) +target_link_libraries(rpc_client PUBLIC rpc_common) +target_include_directories(rpc_client PUBLIC ${PARENT_DIRECTORY}) +set_property(TARGET rpc_client PROPERTY POSITION_INDEPENDENT_CODE ON) +add_library(net::rpc_client ALIAS rpc_client) + +add_library(rpc_server STATIC) +target_sources( + rpc_server + INTERFACE server.hpp + PRIVATE server.cpp +) + +target_link_libraries(rpc_server PUBLIC rpc_common) +target_include_directories(rpc_server PUBLIC ${PARENT_DIRECTORY}) set_property(TARGET rpc_server PROPERTY POSITION_INDEPENDENT_CODE ON) +add_library(net::rpc_server ALIAS rpc_server) + +add_library(net_net STATIC) +target_include_directories(net_net PUBLIC ${PARENT_DIRECTORY}) +target_link_libraries(net_net PUBLIC rpc_client rpc_server) +set_property(TARGET net_net PROPERTY POSITION_INDEPENDENT_CODE ON) +add_library(net::net ALIAS net_net) diff --git a/src/net/client.cpp b/src/net/client.cpp new file mode 100644 index 0000000..c58316b --- /dev/null +++ b/src/net/client.cpp @@ -0,0 +1,59 @@ +/****************************************************************************** + * Copyright 2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of mochi-rpc-server-cxx. + * + * mochi-rpc-server-cxx is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * mochi-rpc-server-cxx is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with mochi-rpc-server-cxx. If not, see + * . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#include +#include +#include "client.hpp" +#include "endpoint.hpp" + +using namespace std::literals; + +namespace network { + + +client::client(const std::string& protocol) + : m_engine(protocol, THALLIUM_CLIENT_MODE) {} + +std::optional +client::lookup(const std::string& address) noexcept { + try { + return endpoint{m_engine, m_engine.lookup(address)}; + } catch(const std::exception& ex) { + LOGGER_ERROR("client::lookup() failed: {}", ex.what()); + return std::nullopt; + } +} + +std::string +client::self_address() const noexcept { + try { + return m_engine.self(); + } catch(const std::exception& ex) { + LOGGER_ERROR("client::self_address() failed: {}", ex.what()); + return "unknown"s; + } +} + +} // namespace network diff --git a/src/net/client.hpp b/src/net/client.hpp new file mode 100644 index 0000000..5e8ebf5 --- /dev/null +++ b/src/net/client.hpp @@ -0,0 +1,51 @@ +/****************************************************************************** + * Copyright 2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of mochi-rpc-server-cxx. + * + * mochi-rpc-server-cxx is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * mochi-rpc-server-cxx is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with mochi-rpc-server-cxx. If not, see + * . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#ifndef NETWORK_CLIENT_HPP +#define NETWORK_CLIENT_HPP + +#include +#include + +namespace network { + +class endpoint; + +class client { + +public: + explicit client(const std::string& protocol); + std::optional + lookup(const std::string& address) noexcept; + std::string + self_address() const noexcept; + +private: + thallium::engine m_engine; +}; + +} // namespace network + +#endif // NETWORK_CLIENT_HPP diff --git a/src/net/endpoint.cpp b/src/net/endpoint.cpp new file mode 100644 index 0000000..4bbd2ce --- /dev/null +++ b/src/net/endpoint.cpp @@ -0,0 +1,40 @@ +/****************************************************************************** + * Copyright 2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of mochi-rpc-server-cxx. + * + * mochi-rpc-server-cxx is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * mochi-rpc-server-cxx is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with mochi-rpc-server-cxx. If not, see + * . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#include "endpoint.hpp" + +#include + +namespace network { + +endpoint::endpoint(thallium::engine& engine, thallium::endpoint endpoint) + : m_engine(engine), m_endpoint(std::move(endpoint)) {} + +std::string +endpoint::address() const { + return m_endpoint; +} + +} // namespace network diff --git a/src/net/endpoint.hpp b/src/net/endpoint.hpp new file mode 100644 index 0000000..ff8764f --- /dev/null +++ b/src/net/endpoint.hpp @@ -0,0 +1,91 @@ +/****************************************************************************** + * Copyright 2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of mochi-rpc-server-cxx. + * + * mochi-rpc-server-cxx is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * mochi-rpc-server-cxx is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with mochi-rpc-server-cxx. If not, see + * . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#ifndef NETWORK_ENDPOINT_HPP +#define NETWORK_ENDPOINT_HPP + +#include +#include +#include + +namespace network { + +enum class rpc_return_policy { requires_response, no_response }; + +class endpoint { + +public: + endpoint(thallium::engine& engine, thallium::endpoint endpoint); + + std::string + address() const; + + template + [[nodiscard]] std::optional> + call(const std::string& rpc_name, Args&&... args) const noexcept + requires(rv == rpc_return_policy::requires_response) { + + try { + const auto rpc = m_engine.define(rpc_name); + return std::make_optional( + rpc.on(m_endpoint)(std::forward(args)...)); + } catch(const std::exception& ex) { + LOGGER_ERROR("endpoint::call() failed: {}", ex.what()); + return std::nullopt; + } + } + + template + void + call(const std::string& rpc_name, Args&&... args) const noexcept + requires(rv == rpc_return_policy::no_response) { + + try { + const auto rpc = m_engine.define(rpc_name).disable_response(); + rpc.on(m_endpoint)(std::forward(args)...); + } catch(const std::exception& ex) { + LOGGER_ERROR("endpoint::call() failed: {}", ex.what()); + } + } + + auto + endp() const { + return m_endpoint; + } + + auto + engine() const { + return m_engine; + } + +private: + mutable thallium::engine m_engine; + thallium::endpoint m_endpoint; +}; + +} // namespace network + +#endif // NETWORK_ENDPOINT_HPP diff --git a/src/net/net/CMakeLists.txt b/src/net/net/CMakeLists.txt deleted file mode 100644 index 1144e16..0000000 --- a/src/net/net/CMakeLists.txt +++ /dev/null @@ -1,33 +0,0 @@ -################################################################################ -# Copyright 2021, Barcelona Supercomputing Center (BSC), Spain # -# # -# This software was partially supported by the EuroHPC-funded project ADMIRE # -# (Project ID: 956748, https://www.admire-eurohpc.eu). # -# # -# This file is part of scord. # -# # -# scord is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# scord is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with scord. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ - -add_library(rpc_server STATIC) -target_sources( - rpc_server - INTERFACE server.hpp - PRIVATE server.cpp -) - -target_link_libraries(rpc_server PUBLIC config logger utils - transport_library Argobots::Argobots Margo::Margo thallium) diff --git a/src/net/request.hpp b/src/net/request.hpp new file mode 100644 index 0000000..74c9898 --- /dev/null +++ b/src/net/request.hpp @@ -0,0 +1,43 @@ +/****************************************************************************** + * Copyright 2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of mochi-rpc-server-cxx. + * + * mochi-rpc-server-cxx is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * mochi-rpc-server-cxx is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with mochi-rpc-server-cxx. If not, see + * . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#ifndef NETWORK_REQUEST_HPP +#define NETWORK_REQUEST_HPP + +#include + +namespace network { + +using request = thallium::request; + +template +inline std::string +get_address(Request&& req) { + return std::forward(req).get_endpoint(); +} + +} // namespace network + +#endif // NETWORK_REQUEST_HPP diff --git a/src/net/serialization.hpp b/src/net/serialization.hpp new file mode 100644 index 0000000..3939fd7 --- /dev/null +++ b/src/net/serialization.hpp @@ -0,0 +1,70 @@ +/****************************************************************************** + * Copyright 2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of mochi-rpc-server-cxx. + * + * mochi-rpc-server-cxx is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * mochi-rpc-server-cxx is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with mochi-rpc-server-cxx. If not, see + * . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#ifndef NETWORK_SERIALIZATION_HPP +#define NETWORK_SERIALIZATION_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +// Cereal does not serialize std::filesystem::path's by default +#include + +namespace cereal { + +//! Loading for std::filesystem::path +template +inline void +CEREAL_LOAD_FUNCTION_NAME(Archive& ar, std::filesystem::path& out) { + std::string tmp; + ar(CEREAL_NVP_("data", tmp)); + out.assign(tmp); +} + +//! Saving for std::filesystem::path +template +inline void +CEREAL_SAVE_FUNCTION_NAME(Archive& ar, const std::filesystem::path& in) { + ar(CEREAL_NVP_("data", in.string())); +} + +} // namespace cereal + +namespace network::serialization { + +#define NETWORK_SERIALIZATION_NVP CEREAL_NVP + +using input_archive = thallium::proc_input_archive<>; +using output_archive = thallium::proc_output_archive<>; + +} // namespace network::serialization + +#endif // NETWORK_SERIALIZATION_HPP diff --git a/src/net/net/server.cpp b/src/net/server.cpp similarity index 63% rename from src/net/net/server.cpp rename to src/net/server.cpp index d06098b..1bca7a0 100644 --- a/src/net/net/server.cpp +++ b/src/net/server.cpp @@ -1,23 +1,23 @@ /****************************************************************************** - * Copyright 2021, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain * * This software was partially supported by the EuroHPC-funded project ADMIRE * (Project ID: 956748, https://www.admire-eurohpc.eu). * - * This file is part of scord. + * This file is part of Cargo. * - * scord is free software: you can redistribute it and/or modify + * Cargo is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * scord is distributed in the hope that it will be useful, + * Cargo is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with scord. If not, see . + * along with Cargo. If not, see . * * SPDX-License-Identifier: GPL-3.0-or-later *****************************************************************************/ @@ -33,18 +33,60 @@ #include #include -#ifdef SCORD_DEBUG_BUILD -#include -#endif - -#include #include -#include +#include "signal_listener.hpp" #include "server.hpp" +#include "endpoint.hpp" using namespace std::literals; -namespace net { +namespace { +void +write_pidfile(const std::filesystem::path& pidfile) { + + int fd; + + if((fd = ::open(pidfile.c_str(), O_RDWR | O_CREAT, 0640)) == -1) { + throw std::system_error(errno, std::system_category(), + "Failed to create daemon lock file"); + } + + if(::lockf(fd, F_TLOCK, 0) < 0) { + throw std::system_error(errno, std::system_category(), + "Failed to acquire lock on pidfile. Another " + "instance of this daemon may already be " + "running"); + } + + /* record pid in lockfile */ + std::string pidstr(std::to_string(getpid())); + + if(::write(fd, pidstr.c_str(), pidstr.length()) != + static_cast(pidstr.length())) { + throw std::system_error(errno, std::system_category(), + "Failed to write pidfile"); + } + + /* flush changes */ + ::fsync(fd); + ::fdatasync(fd); + + ::close(fd); +} +} // namespace + +namespace network { + +server::server(std::string name, std::string address, bool daemonize, + std::filesystem::path rundir, + std::optional pidfile) + + : m_name(std::move(name)), m_address(std::move(address)), + m_daemonize(daemonize), m_rundir(std::move(rundir)), + m_pidfile(daemonize ? std::make_optional(m_rundir / (m_name + ".pid")) + : std::move(pidfile)), + m_logger_config(m_name, logger::logger_type::console_color), + m_network_engine(m_address, THALLIUM_SERVER_MODE) {} server::~server() = default; @@ -63,6 +105,7 @@ server::daemonize() { * Manage signals */ + using fork_event = signal_listener::fork_event; pid_t pid, sid; /* Check if this is already a daemon */ @@ -78,9 +121,10 @@ server::daemonize() { // this (and since we want to be able to output messages from all // processes), we destroy it now and recreate it post-fork() both in the // parent process and in the child. - logger::destroy_global_logger(); + logger::destroy_default_logger(); /* Fork off the parent process */ + m_signal_listener.notify_fork(fork_event::fork_prepare); pid = fork(); // re-initialize logging facilities (post-fork) @@ -93,9 +137,12 @@ server::daemonize() { /* Parent returns to caller */ if(pid != 0) { + m_signal_listener.notify_fork(fork_event::fork_parent); return pid; } + m_signal_listener.notify_fork(fork_event::fork_child); + /* Become a session and process group leader with no controlling tty */ if((sid = setsid()) < 0) { /* Log failure */ @@ -136,34 +183,6 @@ server::daemonize() { exit(EXIT_FAILURE); } - /* Check if daemon already exists: - * First instance of the daemon will lock the file so that other - * instances understand that an instance is already running. - */ - int pfd; - - if((pfd = ::open(m_settings.pidfile().c_str(), O_RDWR | O_CREAT, 0640)) == - -1) { - LOGGER_ERRNO("Failed to create daemon lock file"); - exit(EXIT_FAILURE); - } - - if(::lockf(pfd, F_TLOCK, 0) < 0) { - LOGGER_ERRNO("Failed to acquire lock on pidfile"); - LOGGER_ERROR("Another instance of this daemon may already be running"); - exit(EXIT_FAILURE); - } - - /* record pid in lockfile */ - std::string pidstr(std::to_string(getpid())); - - if(::write(pfd, pidstr.c_str(), pidstr.length()) != - static_cast(pidstr.length())) { - LOGGER_ERRNO("Failed to write pidfile"); - exit(EXIT_FAILURE); - } - - ::close(pfd); ::close(dev_null); /* Manage signals */ @@ -177,11 +196,6 @@ server::daemonize() { return 0; } -config::settings -server::get_configuration() const { - return m_settings; -} - void server::signal_handler(int signum) { @@ -199,6 +213,7 @@ server::signal_handler(int signum) { case SIGHUP: LOGGER_WARN("A signal (SIGHUP) occurred."); + logger::flush_default_logger(); break; default: @@ -208,30 +223,7 @@ server::signal_handler(int signum) { void server::init_logger() { - - if(m_settings.use_console()) { - logger::create_global_logger(m_settings.progname(), "console color"); - return; - } - - if(m_settings.use_syslog()) { - logger::create_global_logger(m_settings.progname(), "syslog"); - - if(!m_settings.daemonize()) { - fmt::print(stderr, "PSA: Output sent to syslog while in " - "non-daemon mode\n"); - } - - return; - } - - if(!m_settings.log_file().empty()) { - logger::create_global_logger(m_settings.progname(), "file", - m_settings.log_file()); - return; - } - - logger::create_global_logger(m_settings.progname(), "console color"); + logger::create_default_logger(m_logger_config); } void @@ -249,12 +241,12 @@ server::install_signal_handlers() { } void -server::check_configuration() {} +server::check_configuration() const {} void server::print_greeting() { - const auto greeting = fmt::format("Starting {} daemon (pid {})", - m_settings.progname(), getpid()); + const auto greeting = + fmt::format("Starting {} daemon (pid {})", m_name, getpid()); LOGGER_INFO("{:=>{}}", "", greeting.size()); LOGGER_INFO(greeting); @@ -262,36 +254,53 @@ server::print_greeting() { } void -server::print_configuration() { +server::print_configuration() const { LOGGER_INFO(""); LOGGER_INFO("[[ Configuration ]]"); - LOGGER_INFO(" - running as daemon?: {}", - (m_settings.daemonize() ? "yes" : "no")); - - if(!m_settings.log_file().empty()) { - LOGGER_INFO(" - log file: {}", m_settings.log_file()); - LOGGER_INFO(" - log file maximum size: {}", - m_settings.log_file_max_size()); - } else { - LOGGER_INFO(" - log file: none"); + LOGGER_INFO(" - running as daemon?: {}", m_daemonize ? "yes" : "no"); + + if(m_logger_config.log_file().has_value()) { + LOGGER_INFO(" - log file: {}", *m_logger_config.log_file()); } - LOGGER_INFO(" - pidfile: {}", m_settings.pidfile()); - LOGGER_INFO(" - port for remote requests: {}", m_settings.remote_port()); - LOGGER_INFO(" - workers: {}", m_settings.workers_in_pool()); + if(m_pidfile.has_value()) { + LOGGER_INFO(" - pidfile: {}", *m_pidfile); + } + + LOGGER_INFO(" - address for remote requests: {}", self_address()); LOGGER_INFO(""); } void server::print_farewell() { - const auto farewell = fmt::format("Stopping {} daemon (pid {})", - m_settings.progname(), getpid()); + const auto farewell = + fmt::format("Stopping {} daemon (pid {})", m_name, getpid()); LOGGER_INFO("{:=>{}}", "", farewell.size()); LOGGER_INFO(farewell); LOGGER_INFO("{:=>{}}", "", farewell.size()); } +std::optional +server::lookup(const std::string& address) noexcept { + try { + return endpoint{m_network_engine, m_network_engine.lookup(address)}; + } catch(const std::exception& ex) { + LOGGER_ERROR("server::lookup() failed: {}", ex.what()); + return std::nullopt; + } +} + +std::string +server::self_address() const noexcept { + try { + return m_network_engine.self(); + } catch(const std::exception& ex) { + LOGGER_ERROR("server::self_address() failed: {}", ex.what()); + return "unknown"s; + } +} + int server::run() { @@ -301,20 +310,23 @@ server::run() { // validate settings check_configuration(); -#ifdef SCORD_DEBUG_BUILD - if(::prctl(PR_SET_DUMPABLE, 1) != 0) { - LOGGER_WARN("Failed to set PR_SET_DUMPABLE flag for process. " - "Daemon will not produce core dumps."); - } -#endif - // daemonize if needed - if(m_settings.daemonize() && daemonize() != 0) { - /* parent clean ups and exits, child continues */ - teardown(); + if(m_daemonize && daemonize() != 0) { + /* parent exits, child continues */ + shutdown(); return EXIT_SUCCESS; } + // write pidfile if needed + if(m_pidfile.has_value()) { + try { + ::write_pidfile(m_pidfile.value()); + } catch(const std::system_error& e) { + LOGGER_ERROR("Failed to create pidfile: {}", e.what()); + return EXIT_FAILURE; + } + } + // print useful information print_greeting(); print_configuration(); @@ -345,11 +357,16 @@ server::teardown() { LOGGER_INFO("* Stopping signal listener..."); m_signal_listener.stop(); + if(!m_daemonize) { + return; + } + + LOGGER_INFO("* Removing pidfile..."); std::error_code ec; - fs::remove(m_settings.pidfile(), ec); + std::filesystem::remove(*m_pidfile, ec); if(ec) { - LOGGER_ERROR("Failed to remove pidfile {}: {}", m_settings.pidfile(), + LOGGER_ERROR("Failed to remove pidfile {}: {}", *m_pidfile, ec.message()); } } @@ -365,4 +382,4 @@ server::shutdown() { m_network_engine.finalize(); } -} // namespace net +} // namespace network diff --git a/src/net/net/server.hpp b/src/net/server.hpp similarity index 60% rename from src/net/net/server.hpp rename to src/net/server.hpp index c264809..5d5e66c 100644 --- a/src/net/net/server.hpp +++ b/src/net/server.hpp @@ -22,50 +22,48 @@ * SPDX-License-Identifier: GPL-3.0-or-later *****************************************************************************/ -#ifndef SERVER_HPP -#define SERVER_HPP +#ifndef RPC_SERVER_HPP +#define RPC_SERVER_HPP -#include +#include +#include +#include #include #include #include +#include "signal_listener.hpp" -namespace config { -struct settings; -} // namespace config +namespace network { -namespace utils { -struct signal_listener; -} // namespace utils - -namespace net { +class endpoint; using request = thallium::request; +template +using provider = thallium::provider; + class server { public: - template - explicit server(config::settings cfg, Handlers&&... handlers) - : m_settings(std::move(cfg)) { - - using namespace std::literals; - - const std::string thallim_address = - m_settings.transport_protocol() + "://"s + - m_settings.bind_address() + ":"s + - std::to_string(m_settings.remote_port()); + server(std::string name, std::string address, bool daemonize, + std::filesystem::path rundir, + std::optional pidfile = {}); - m_network_engine = - thallium::engine(thallim_address, THALLIUM_SERVER_MODE); + ~server(); - (set_handler(std::forward(handlers)), ...); + template + void + configure_logger(logger::logger_type type, Args&&... args) { + m_logger_config = logger::logger_config(m_name, type, + std::forward(args)...); } - ~server(); + std::optional + lookup(const std::string& address) noexcept; + + std::string + self_address() const noexcept; - config::settings - get_configuration() const; int run(); void @@ -86,28 +84,38 @@ private: daemonize(); void signal_handler(int); - void init_logger(); void install_signal_handlers(); - - void - check_configuration(); void print_greeting(); void - print_configuration(); - void print_farewell(); +protected: + virtual void + check_configuration() const; + + virtual void + print_configuration() const; + private: - config::settings m_settings; + std::string m_name; + std::string m_address; + bool m_daemonize; + std::filesystem::path m_rundir; + std::optional m_pidfile; + logger::logger_config m_logger_config; + +protected: thallium::engine m_network_engine; - utils::signal_listener m_signal_listener; + +private: + signal_listener m_signal_listener; }; -} // namespace net +} // namespace network -#endif // SERVER_HPP +#endif // RPC_SERVER_HPP diff --git a/src/net/signal_listener.hpp b/src/net/signal_listener.hpp new file mode 100644 index 0000000..0aa2ac1 --- /dev/null +++ b/src/net/signal_listener.hpp @@ -0,0 +1,121 @@ +/****************************************************************************** + * Copyright 2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of mochi-rpc-server-cxx. + * + * mochi-rpc-server-cxx is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * mochi-rpc-server-cxx is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with mochi-rpc-server-cxx. If not, see + * . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#ifndef SIGNAL_LISTENER_HPP +#define SIGNAL_LISTENER_HPP + +#include +#include + +namespace { +template +void +do_for(F f, Args... args) { + [[maybe_unused]] int x[] = {(f(args), 0)...}; +} +} // namespace + +struct signal_listener { + + using fork_event = asio::execution_context::fork_event; + + using SignalHandlerType = std::function; + + signal_listener() : m_ios(), m_signals(m_ios) {} + + ~signal_listener() { + m_signals.cancel(); + m_ios.stop(); + if(m_thread.joinable()) { + m_thread.join(); + } + } + + template + void + set_handler(SignalHandlerType&& handler, Args... signums) { + + m_user_handler = std::forward(handler); + m_signals.clear(); + + ::do_for([&](int signum) { m_signals.add(signum); }, signums...); + } + + void + clear_handler() { + m_user_handler = nullptr; + m_signals.clear(); + } + + void + do_accept() { + + if(m_user_handler) { + m_signals.async_wait(std::bind( // NOLINT + &signal_listener::signal_handler, this, + std::placeholders::_1, std::placeholders::_2)); + } + } + + void + run() { + m_thread = std::thread([&]() { + do_accept(); + m_ios.run(); + }); + } + + void + notify_fork(signal_listener::fork_event event) { + m_ios.notify_fork(event); + } + + void + stop() { + m_ios.stop(); + } + +private: + void + signal_handler(asio::error_code ec, int signal_number) { + // a signal occurred, invoke installed handler + if(!ec) { + m_user_handler(signal_number); + + // reinstall handler + m_signals.async_wait( + std::bind(&signal_listener::signal_handler, this, // NOLINT + std::placeholders::_1, std::placeholders::_2)); + } + } + + + asio::io_service m_ios; + asio::signal_set m_signals; + std::thread m_thread; + SignalHandlerType m_user_handler; +}; + +#endif // SIGNAL_LISTENER_HPP diff --git a/src/net/utilities.hpp b/src/net/utilities.hpp new file mode 100644 index 0000000..4e01d71 --- /dev/null +++ b/src/net/utilities.hpp @@ -0,0 +1,136 @@ +/****************************************************************************** + * Copyright 2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of mochi-rpc-server-cxx. + * + * mochi-rpc-server-cxx is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * mochi-rpc-server-cxx is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with mochi-rpc-server-cxx. If not, see + * . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + + +#ifndef NETWORK_UTILITIES_HPP +#define NETWORK_UTILITIES_HPP + +#include +#include +#include +#include +#include +#include + +namespace network { + +class rpc_info { +private: + static std::uint64_t + new_id() { + static std::atomic_uint64_t s_current_id; + return s_current_id++; + } + +public: + rpc_info(std::uint64_t id, std::string name, std::string address) + : m_id(id), m_children(0), m_name(std::move(name)), + m_address(std::move(address)) {} + + rpc_info(std::uint64_t id, std::uint64_t pid, std::string name, + std::string address) + : m_id(id), m_pid(pid), m_children(0), m_name(std::move(name)), + m_address(std::move(address)) {} + + template + static rpc_info + create(Args&&... args) { + return {new_id(), std::forward(args)...}; + } + + template + rpc_info + add_child(std::string address) const { + return {m_children, m_id, m_name, std::move(address)}; + } + + constexpr std::uint64_t + id() const { + return m_id; + } + + constexpr std::optional + pid() const { + return m_pid; + } + + const std::string& + name() const { + return m_name; + } + + const std::string& + address() const { + return m_address; + } + +private: + std::uint64_t m_id; + std::optional m_pid; + std::uint64_t m_children; + std::string m_name; + std::string m_address; +}; + +} // namespace network + +template <> +struct fmt::formatter { + + // RPC direction format: + // '<': from self to target (outbound) + // '>': from target to self (inbound) + bool m_outbound = true; + + // Parses format specifications in the form: + constexpr auto + parse(format_parse_context& ctx) { + auto it = ctx.begin(), end = ctx.end(); + + if(it != end && (*it == '<' || *it == '>')) { + m_outbound = *it++ == '<'; + } + + if(it != end && *it != '}') { + ctx.on_error("invalid format"); + } + + return it; + } + + template + constexpr auto + format(const network::rpc_info& rpc, FormatContext& ctx) const { + format_to(ctx.out(), "{}{} id: {} name: {} ", m_outbound ? "<=" : "=>", + rpc.pid() ? fmt::format(" pid: {}", *rpc.pid()) : "", + rpc.id(), std::quoted(rpc.name())); + return m_outbound ? format_to(ctx.out(), "to: {}", + std::quoted(rpc.address())) + : format_to(ctx.out(), "from: {}", + std::quoted(rpc.address())); + } +}; + +#endif // NETWORK_UTILITIES_HPP diff --git a/src/worker.cpp b/src/worker.cpp index 5772198..c4aebda 100644 --- a/src/worker.cpp +++ b/src/worker.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include "message.hpp" -- GitLab From 30c8e586060c1b08baf98188f2f4a00985f0a907 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 31 Aug 2023 17:27:14 +0200 Subject: [PATCH 3/6] Remove `config` module --- src/config/CMakeLists.txt | 34 ---- src/config/config/CMakeLists.txt | 84 --------- src/config/config/config.hpp | 31 ---- src/config/config/defaults.cpp.in | 52 ------ src/config/config/defaults.hpp | 53 ------ src/config/config/file_options.yml | 41 ----- src/config/config/genopts.yml.in | 14 -- src/config/config/parsers.cpp | 103 ----------- src/config/config/parsers.hpp | 49 ----- src/config/config/settings.cpp | 278 ----------------------------- src/config/config/settings.hpp | 232 ------------------------ 11 files changed, 971 deletions(-) delete mode 100644 src/config/CMakeLists.txt delete mode 100644 src/config/config/CMakeLists.txt delete mode 100644 src/config/config/config.hpp delete mode 100644 src/config/config/defaults.cpp.in delete mode 100644 src/config/config/defaults.hpp delete mode 100644 src/config/config/file_options.yml delete mode 100644 src/config/config/genopts.yml.in delete mode 100644 src/config/config/parsers.cpp delete mode 100644 src/config/config/parsers.hpp delete mode 100644 src/config/config/settings.cpp delete mode 100644 src/config/config/settings.hpp diff --git a/src/config/CMakeLists.txt b/src/config/CMakeLists.txt deleted file mode 100644 index 86f5351..0000000 --- a/src/config/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -################################################################################ -# Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain # -# # -# This software was partially supported by the EuroHPC-funded project ADMIRE # -# (Project ID: 956748, https://www.admire-eurohpc.eu). # -# # -# This file is part of Cargo. # -# # -# Cargo is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# Cargo is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with Cargo. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ - -add_subdirectory(config) - -# Since some of the sources will be auto-generated, we need to search for -# includes in ${CMAKE_CURRENT_BINARY_DIR} -target_include_directories( - config PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_BINARY_DIR} -) - -set_property(TARGET config PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/src/config/config/CMakeLists.txt b/src/config/config/CMakeLists.txt deleted file mode 100644 index a6d95ec..0000000 --- a/src/config/config/CMakeLists.txt +++ /dev/null @@ -1,84 +0,0 @@ -################################################################################ -# Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain # -# # -# This software was partially supported by the EuroHPC-funded project ADMIRE # -# (Project ID: 956748, https://www.admire-eurohpc.eu). # -# # -# This file is part of Cargo. # -# # -# Cargo is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# Cargo is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with Cargo. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ - -# Create a config target for all configuration code -add_library(config STATIC) - -target_include_directories( - config PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} -) - -target_sources( - config - PRIVATE config.hpp - defaults.hpp - ${CMAKE_CURRENT_BINARY_DIR}/defaults.cpp - ${CMAKE_CURRENT_BINARY_DIR}/config_options.hpp - ${CMAKE_CURRENT_BINARY_DIR}/keywords.hpp - parsers.cpp - parsers.hpp - settings.cpp - settings.hpp -) - -target_link_libraries(config PRIVATE utils file_options::file_options) - -# ############################################################################## -# Produce several auto-generated files for 'config' -# ############################################################################## - -# Default values for options -configure_file(defaults.cpp.in defaults.cpp) - -# Automatic generation of file_options schemas. To facilitate the management of -# any file-based configuration options, we generate a C++ schema from a YAML -# file that allows the 'file_options' library to parse them fromm a -# configuration file. The process works as follows: -# 1. We define the desired options in 'file_options.yml' -# 2. We rely on the 'genopts' tool to generate valid C++ schemas for the -# 'file_options' library. This tool can be configured using a 'genopts.yml' -# configuration file. -# 3. In the daemon code, we use the facilities provided by the 'file_options' -# library to access the options values. - -# Since the configuration in genopts.yml can be path-dependant, we rely on -# CMake to substitute any special @variables@ for their actual values -configure_file(genopts.yml.in genopts.yml @ONLY) - -# Define the command that will generate config_options.hpp and keywords.hpp. -# It will be executed since there is a direct dependency between 'config' -# (defined below) and these output files. -# We also make the command depend on file_options.yml and genopts.yml so that -# it gets re-executed if they change. -add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/config_options.hpp - ${CMAKE_CURRENT_BINARY_DIR}/keywords.hpp - COMMENT "Generating config_options.hpp, keywords.hpp" - DEPENDS genopts_virtualenv file_options.yml - ${CMAKE_CURRENT_BINARY_DIR}/genopts.yml - COMMAND - Genopts::Python3_Interpreter -m genopts --config - ${CMAKE_CURRENT_BINARY_DIR}/genopts.yml - ${CMAKE_CURRENT_LIST_DIR}/file_options.yml -) diff --git a/src/config/config/config.hpp b/src/config/config/config.hpp deleted file mode 100644 index 56d1fe0..0000000 --- a/src/config/config/config.hpp +++ /dev/null @@ -1,31 +0,0 @@ -/****************************************************************************** - * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of Cargo. - * - * Cargo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Cargo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Cargo. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#ifndef CONFIG_HPP -#define CONFIG_HPP - -#include "defaults.hpp" -#include "settings.hpp" - -#endif /* CONFIG_HPP */ diff --git a/src/config/config/defaults.cpp.in b/src/config/config/defaults.cpp.in deleted file mode 100644 index 122ad3c..0000000 --- a/src/config/config/defaults.cpp.in +++ /dev/null @@ -1,52 +0,0 @@ -/****************************************************************************** - * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of Cargo. - * - * Cargo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Cargo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Cargo. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -/* DO NOT MODIFY: This file is autogenerated by CMake */ -#include -#include -#include -#include -#include - -namespace config::defaults { - -const char* progname = "@CMAKE_PROJECT_NAME@"; -const bool daemonize = true; -const bool use_syslog = false; -const bool use_console = false; -const std::filesystem::path log_file = {}; -const uint32_t log_file_max_size = static_cast(16 * 1024 * 1024); - -const char* transport_protocol = "ofi+tcp"; -const char* bind_address = "127.0.0.1"; -const in_port_t remote_port = 42000; -const char* pidfile = - "@CMAKE_INSTALL_FULL_LOCALSTATEDIR@/@CMAKE_PROJECT_NAME@.pid"; - -const uint32_t workers_in_pool = std::thread::hardware_concurrency(); -const uint32_t backlog_size = 128; -const char* config_file = - "@CMAKE_INSTALL_FULL_SYSCONFDIR@/@CMAKE_PROJECT_NAME@.conf"; - -} // namespace config::defaults diff --git a/src/config/config/defaults.hpp b/src/config/config/defaults.hpp deleted file mode 100644 index 0126be2..0000000 --- a/src/config/config/defaults.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/****************************************************************************** - * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of Cargo. - * - * Cargo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Cargo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Cargo. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#ifndef CONFIG_DEFAULTS_HPP -#define CONFIG_DEFAULTS_HPP - -#include -#include -#include - -namespace fs = std::filesystem; - -namespace config::defaults { - -extern const char* progname; -extern const bool daemonize; -extern const bool use_syslog; -extern const bool use_console; -extern const fs::path log_file; -extern const uint32_t log_file_max_size; -extern const char* transport_protocol; -extern const char* bind_address; -extern const in_port_t remote_port; -extern const char* pidfile; -extern const uint32_t workers_in_pool; -extern const char* staging_directory; -extern const uint32_t backlog_size; -extern const char* config_file; - -} // namespace config::defaults - -#endif /* CONFIG_DEFAULTS_HPP */ diff --git a/src/config/config/file_options.yml b/src/config/config/file_options.yml deleted file mode 100644 index 7049676..0000000 --- a/src/config/config/file_options.yml +++ /dev/null @@ -1,41 +0,0 @@ -sections: - - name: "global_settings" - required: true - options: - - name: "use_syslog" - required: true - type: "bool" - converter: "parsers::parse_bool" - - - name: "log_file" - required: false - type: "std::filesystem::path" - converter: "parsers::parse_path" - - - name: "log_file_max_size" - required: false - type: "uint32_t" - converter: "parsers::parse_capacity" - - - name: "transport_protocol" - required: true - type: "std::string" - - - name: "bind_address" - required: true - type: "std::string" - - - name: "remote_port" - required: true - type: "uint32_t" - converter: "parsers::parse_number" - - - name: "pidfile" - required: true - type: "std::filesystem::path" - converter: "parsers::parse_path" - - - name: "workers" - required: true - type: "uint32_t" - converter: "parsers::parse_number" diff --git a/src/config/config/genopts.yml.in b/src/config/config/genopts.yml.in deleted file mode 100644 index 391c257..0000000 --- a/src/config/config/genopts.yml.in +++ /dev/null @@ -1,14 +0,0 @@ -genopts: - schema: - copyright_file: "@CMAKE_SOURCE_DIR@/COPYRIGHT_NOTICE" - header_guard: "CONFIG_SCHEMA_HPP" - namespace: "config" - output_path: - header: "@CMAKE_CURRENT_BINARY_DIR@/config_options.hpp" - - keywords: - copyright_file: "@CMAKE_SOURCE_DIR@/COPYRIGHT_NOTICE" - header_guard: "CONFIG_KEYWORDS_HPP" - namespace: "config::keywords" - output_path: - header: "@CMAKE_CURRENT_BINARY_DIR@/keywords.hpp" diff --git a/src/config/config/parsers.cpp b/src/config/config/parsers.cpp deleted file mode 100644 index 4c6164b..0000000 --- a/src/config/config/parsers.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/****************************************************************************** - * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of Cargo. - * - * Cargo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Cargo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Cargo. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include "parsers.hpp" - -namespace fs = std::filesystem; - -namespace config::parsers { - -bool -parse_bool(const std::string& name, const std::string& value) { - - if(value == "1" || boost::algorithm::to_lower_copy(value) == "true") { - return true; - } - - if(value == "0" || boost::algorithm::to_lower_copy(value) == "false") { - return false; - } - - throw std::invalid_argument("Value provided for option '" + name + - "' is not boolean"); -} - -uint32_t -parse_number(const std::string& name, const std::string& value) { - - int32_t optval = 0; - - try { - optval = std::stoi(value); - } catch(...) { - throw std::invalid_argument("Value provided for option '" + name + - "' is not a number"); - } - - if(optval <= 0) { - throw std::invalid_argument("Value provided for option '" + name + - "' must be greater than zero"); - } - - return static_cast(optval); -} - -fs::path -parse_path(const std::string& name, const std::string& value) { - - (void) name; - - return {value}; -} - -fs::path -parse_existing_path(const std::string& name, const std::string& value) { - - if(!fs::exists(value)) { - throw std::invalid_argument("Path '" + value + "' in option '" + name + - "' does not exist"); - } - - return {value}; -} - -uint64_t -parse_capacity(const std::string& name, const std::string& value) { - - try { - return utils::parse_size(value); - } catch(const std::exception& e) { - throw std::invalid_argument("Value provided in option '" + name + - "' is invalid"); - } -} - -} // namespace config::parsers diff --git a/src/config/config/parsers.hpp b/src/config/config/parsers.hpp deleted file mode 100644 index 5a23471..0000000 --- a/src/config/config/parsers.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/****************************************************************************** - * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of Cargo. - * - * Cargo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Cargo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Cargo. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#ifndef CONFIG_PARSERS_HPP -#define CONFIG_PARSERS_HPP - -#include -#include -#include - -namespace fs = std::filesystem; - -namespace config::parsers { - -bool -parse_bool(const std::string& name, const std::string& value); -uint32_t -parse_number(const std::string& name, const std::string& value); -fs::path -parse_path(const std::string& name, const std::string& value); -fs::path -parse_existing_path(const std::string& name, const std::string& value); -uint64_t -parse_capacity(const std::string& name, const std::string& value); - -} // namespace config::parsers - -#endif // CONFIG_PARSERS_HPP diff --git a/src/config/config/settings.cpp b/src/config/config/settings.cpp deleted file mode 100644 index 8f2916c..0000000 --- a/src/config/config/settings.cpp +++ /dev/null @@ -1,278 +0,0 @@ -/****************************************************************************** - * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of Cargo. - * - * Cargo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Cargo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Cargo. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#include -#include -#include "config_options.hpp" -#include "defaults.hpp" -#include "file_options/options_description.hpp" -#include "file_options/yaml_parser.hpp" -#include "keywords.hpp" -#include "settings.hpp" - -namespace fs = std::filesystem; - -namespace config { - -settings::settings() = default; - -settings::settings(std::string progname, bool daemonize, bool use_syslog, - bool use_console, fs::path log_file, - const uint32_t log_file_max_size, - std::string transport_protocol, std::string bind_address, - uint32_t remote_port, fs::path pidfile, uint32_t workers, - uint32_t backlog_size, fs::path cfgfile, - std::list defns) - : m_progname(std::move(progname)), m_daemonize(daemonize), - m_use_syslog(use_syslog), m_use_console(use_console), - m_log_file(std::move(log_file)), m_log_file_max_size(log_file_max_size), - m_transport_protocol(std::move(transport_protocol)), - m_bind_address(std::move(bind_address)), m_remote_port(remote_port), - m_daemon_pidfile(std::move(pidfile)), m_workers_in_pool(workers), - m_backlog_size(backlog_size), m_config_file(std::move(cfgfile)), - m_default_namespaces(std::move(defns)) {} - -void -settings::load_defaults() { - m_progname = defaults::progname; - m_daemonize = defaults::daemonize; - m_use_syslog = defaults::use_syslog; - m_use_console = defaults::use_console; - m_log_file = defaults::log_file; - m_log_file_max_size = defaults::log_file_max_size; - m_transport_protocol = defaults::transport_protocol; - m_bind_address = defaults::bind_address; - m_remote_port = defaults::remote_port; - m_daemon_pidfile = defaults::pidfile; - m_workers_in_pool = defaults::workers_in_pool; - m_backlog_size = defaults::backlog_size; - m_config_file = defaults::config_file; - m_default_namespaces.clear(); -} - -void -settings::load_from_file(const fs::path& filename) { - file_options::options_map opt_map; - file_options::parse_yaml_file(filename, config::valid_options, opt_map); - - // load global settings - const auto& gsettings = opt_map.get_as( - keywords::global_settings); - - m_progname = defaults::progname; - m_use_syslog = gsettings.get_as(keywords::use_syslog); - m_use_console = defaults::use_console; - - if(gsettings.has(keywords::log_file)) { - m_log_file = gsettings.get_as(keywords::log_file); - } - - if(gsettings.has(keywords::log_file_max_size)) { - m_log_file_max_size = - gsettings.get_as(keywords::log_file_max_size); - } - - m_transport_protocol = - gsettings.get_as(keywords::transport_protocol); - m_bind_address = gsettings.get_as(keywords::bind_address); - m_remote_port = gsettings.get_as(keywords::remote_port); - m_daemon_pidfile = gsettings.get_as(keywords::pidfile); - m_workers_in_pool = gsettings.get_as(keywords::workers); - m_backlog_size = defaults::backlog_size; - - // load definitions for default namespaces - // const auto& namespaces = - // opt_map.get_as(keywords::namespaces); - // - // for(const auto& nsdef : namespaces) { - // m_default_namespaces.emplace_back( - // nsdef.get_as(keywords::nsid), - // nsdef.get_as(keywords::track_contents), - // nsdef.get_as(keywords::mountpoint), - // nsdef.get_as(keywords::type), - // nsdef.get_as(keywords::capacity), - // nsdef.get_as(keywords::visibility)); - // } -} - -std::string -settings::to_string() const { - std::string str = - std::string("settings {\n") + " m_progname: " + m_progname + - ",\n" + " m_daemonize: " + (m_daemonize ? "true" : "false") + - ",\n" + " m_use_syslog: " + (m_use_syslog ? "true" : "false") + - ",\n" + " m_use_console: " + (m_use_console ? "true" : "false") + - ",\n" + " m_log_file: " + m_log_file.string() + ",\n" + - " m_log_file_max_size: " + std::to_string(m_log_file_max_size) + - ",\n" + " m_bind_address: " + m_bind_address + ",\n" + - " m_remote_port: " + std::to_string(m_remote_port) + ",\n" + - " m_pidfile: " + m_daemon_pidfile.string() + ",\n" + - " m_workers: " + std::to_string(m_workers_in_pool) + ",\n" + - " m_backlog_size: " + std::to_string(m_backlog_size) + ",\n" + - " m_config_file: " + m_config_file.string() + ",\n" + "};"; - // TODO: add m_default_namespaces - return str; -} - -std::string -settings::progname() const { - return m_progname; -} - -void -settings::progname(const std::string& progname) { - m_progname = progname; -} - -bool -settings::daemonize() const { - return m_daemonize; -} - -void -settings::daemonize(bool daemonize) { - m_daemonize = daemonize; -} - -bool -settings::use_syslog() const { - return m_use_syslog; -} - -void -settings::use_syslog(bool use_syslog) { - m_use_syslog = use_syslog; -} - -bool -settings::use_console() const { - return m_use_console; -} - -void -settings::use_console(bool use_console) { - m_use_console = use_console; -} - -fs::path -settings::log_file() const { - return m_log_file; -} - -void -settings::log_file(const fs::path& log_file) { - m_log_file = log_file; -} - -uint32_t -settings::log_file_max_size() const { - return m_log_file_max_size; -} - -void -settings::log_file_max_size(uint32_t log_file_max_size) { - m_log_file_max_size = log_file_max_size; -} - -std::string -settings::transport_protocol() const { - return m_transport_protocol; -} -void -settings::transport_protocol(const std::string& transport_protocol) { - m_transport_protocol = transport_protocol; -} - -std::string -settings::bind_address() const { - return m_bind_address; -} - -void -settings::bind_address(const std::string& bind_address) { - m_bind_address = bind_address; -} - -in_port_t -settings::remote_port() const { - return m_remote_port; -} - -void -settings::remote_port(in_port_t remote_port) { - m_remote_port = remote_port; -} - -fs::path -settings::pidfile() const { - return m_daemon_pidfile; -} - -void -settings::pidfile(const fs::path& pidfile) { - m_daemon_pidfile = pidfile; -} - -uint32_t -settings::workers_in_pool() const { - return m_workers_in_pool; -} - -void -settings::workers_in_pool(uint32_t workers_in_pool) { - m_workers_in_pool = workers_in_pool; -} - -uint32_t -settings::backlog_size() const { - return m_backlog_size; -} - -void -settings::backlog_size(uint32_t backlog_size) { - m_backlog_size = backlog_size; -} - -fs::path -settings::config_file() const { - return m_config_file; -} - -void -settings::config_file(const fs::path& config_file) { - m_config_file = config_file; -} - -std::list -settings::default_namespaces() const { - return m_default_namespaces; -} - -void -settings::default_namespaces( - const std::list& default_namespaces) { - m_default_namespaces = default_namespaces; -} - -} // namespace config diff --git a/src/config/config/settings.hpp b/src/config/config/settings.hpp deleted file mode 100644 index 8d10670..0000000 --- a/src/config/config/settings.hpp +++ /dev/null @@ -1,232 +0,0 @@ -/****************************************************************************** - * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of scord. - * - * scord is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * scord is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with scord. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#ifndef CONFIG_SETTINGS_HPP -#define CONFIG_SETTINGS_HPP - -#include -#include -#include -#include -#include -#include - -#include "defaults.hpp" - -namespace fs = std::filesystem; - -namespace config { - -struct namespace_def { - - namespace_def(std::string nsid, bool track, fs::path mountpoint, - std::string alias, const uint64_t capacity, - std::string visibility) - : m_nsid(std::move(nsid)), m_track(track), - m_mountpoint(std::move(mountpoint)), m_alias(std::move(alias)), - m_capacity(capacity), m_visibility(std::move(visibility)) {} - - namespace_def(const namespace_def& other) = default; - - namespace_def(namespace_def&& rhs) = default; - - namespace_def& - operator=(const namespace_def& other) = default; - - namespace_def& - operator=(namespace_def&& rhs) = default; - - std::string - nsid() const { - return m_nsid; - } - - bool - track() const { - return m_track; - } - - fs::path - mountpoint() const { - return m_mountpoint; - } - - std::string - alias() const { - return m_alias; - } - - uint64_t - capacity() const { - return m_capacity; - } - - std::string - visibility() const { - return m_visibility; - } - - std::string m_nsid; - bool m_track; - fs::path m_mountpoint; - std::string m_alias; - uint64_t m_capacity; - std::string m_visibility; -}; - -struct settings { - - settings(); - - settings(std::string progname, bool daemonize, bool use_syslog, - bool use_console, fs::path log_file, - const uint32_t log_file_max_size, std::string transport_protocol, - std::string bind_address, uint32_t remote_port, fs::path pidfile, - uint32_t workers, uint32_t backlog_size, fs::path cfgfile, - std::list defns); - - void - load_defaults(); - - void - load_from_file(const fs::path& filename); - - std::string - to_string() const; - - settings(const settings& other) = default; - - settings(settings&& rhs) = default; - - settings& - operator=(const settings& other) = default; - - settings& - operator=(settings&& rhs) = default; - - ~settings() = default; - - std::string - progname() const; - - void - progname(const std::string& progname); - - bool - daemonize() const; - - void - daemonize(bool daemonize); - - bool - use_syslog() const; - - void - use_syslog(bool use_syslog); - - bool - use_console() const; - - void - use_console(bool use_console); - - fs::path - log_file() const; - - void - log_file(const fs::path& log_file); - - uint32_t - log_file_max_size() const; - - void - log_file_max_size(uint32_t log_file_max_size); - - std::string - transport_protocol() const; - - void - transport_protocol(const std::string& transport_protocol); - - std::string - bind_address() const; - - void - bind_address(const std::string& bind_address); - - in_port_t - remote_port() const; - - void - remote_port(in_port_t remote_port); - - fs::path - pidfile() const; - - void - pidfile(const fs::path& pidfile); - - uint32_t - workers_in_pool() const; - - void - workers_in_pool(uint32_t workers_in_pool); - - uint32_t - backlog_size() const; - - void - backlog_size(uint32_t backlog_size); - - fs::path - config_file() const; - - void - config_file(const fs::path& config_file); - - std::list - default_namespaces() const; - - void - default_namespaces(const std::list& default_namespaces); - - std::string m_progname = defaults::progname; - bool m_daemonize = defaults::daemonize; - bool m_use_syslog = defaults::use_syslog; - bool m_use_console = defaults::use_console; - fs::path m_log_file = defaults::log_file; - uint32_t m_log_file_max_size = defaults::log_file_max_size; - std::string m_transport_protocol = defaults::transport_protocol; - std::string m_bind_address = defaults::bind_address; - in_port_t m_remote_port = defaults::remote_port; - fs::path m_daemon_pidfile = defaults::pidfile; - uint32_t m_workers_in_pool = defaults::workers_in_pool; - uint32_t m_backlog_size = defaults::backlog_size; - fs::path m_config_file = defaults::config_file; - std::list m_default_namespaces; -}; - -} // namespace config - -#endif /* CONFIG_SETTINGS_HPP */ -- GitLab From 4744b45fd6961b7125109ada0ec27a6720672ecd Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 31 Aug 2023 17:28:52 +0200 Subject: [PATCH 4/6] Handle CLI args with CLI11 Remove boost::program_options --- CMakeLists.txt | 37 +++-------- src/CMakeLists.txt | 2 +- src/cargo.cpp | 161 +++++++++++---------------------------------- src/master.cpp | 1 - 4 files changed, 50 insertions(+), 151 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f1ac6da..8e61830 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,9 +159,9 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) ### instead, make sure that pkg-config is available find_package(PkgConfig REQUIRED) -### boost libraries: required for processing program options +### boost libraries: required for boost::mpi message(STATUS "[${PROJECT_NAME}] Checking for boost libraries") -find_package(Boost 1.53 REQUIRED COMPONENTS program_options mpi) +find_package(Boost 1.53 REQUIRED COMPONENTS mpi) ### transport library if (CARGO_TRANSPORT_LIBRARY STREQUAL libfabric) @@ -216,35 +216,18 @@ FetchContent_Declare( FetchContent_MakeAvailable(spdlog) set_target_properties(spdlog PROPERTIES POSITION_INDEPENDENT_CODE ON) -### file_options: required for reading configuration files -message(STATUS "[${PROJECT_NAME}] Downloading and building file_options") -FetchContent_Declare( - file_options - GIT_REPOSITORY https://storage.bsc.es/gitlab/utils/file_options - GIT_TAG bdb4f7f7f2dd731815241fc41afe6373df8f732a # v0.1.0-pre - GIT_SHALLOW ON - GIT_PROGRESS ON -) - -FetchContent_MakeAvailable(file_options) - -### genopts: required for generating file_options schemas -message(STATUS "[${PROJECT_NAME}] Downloading and building genopts") -FetchContent_Declare( - genopts - GIT_REPOSITORY https://storage.bsc.es/gitlab/utils/genopts - GIT_TAG c456c2d8ec92f26d9074b123446261103e5c847c # v0.1.0-pre - # enabling GIT_SHALLOW when the GIT_TAG value is not backed by an - # actual tag is problematic - GIT_SHALLOW OFF - GIT_PROGRESS ON +### CLI11: used for parsing command-line options +message(STATUS "[${PROJECT_NAME}] Searching for CLI11") +FetchContent_Declare(cli11 +GIT_REPOSITORY https://github.com/CLIUtils/CLI11 +GIT_TAG 291c58789c031208f08f4f261a858b5b7083e8e2 # v2.3.2 +GIT_SHALLOW ON +GIT_PROGRESS ON ) - -FetchContent_MakeAvailable(genopts) +FetchContent_MakeAvailable(cli11) ### expected: required for using tl::expected in the C++ library implementation ### until std::expected makes it to C++ - message(STATUS "[${PROJECT_NAME}] Downloading and building tl::expected") set(EXPECTED_BUILD_PACKAGE OFF) set(EXPECTED_BUILD_TESTS OFF) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2f34620..c3538e7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -55,7 +55,7 @@ target_link_libraries( cargo fmt::fmt MPI::MPI_CXX - Boost::program_options + CLI11::CLI11 Boost::serialization Boost::mpi posix_file diff --git a/src/cargo.cpp b/src/cargo.cpp index 8de01f5..e752637 100644 --- a/src/cargo.cpp +++ b/src/cargo.cpp @@ -24,154 +24,71 @@ #include -#include #include #include #include #include #include #include +#include +#include #include -#include -#include #include "master.hpp" #include "worker.hpp" #include "env.hpp" namespace fs = std::filesystem; -namespace bpo = boost::program_options; namespace mpi = boost::mpi; // utility functions namespace { -void -print_version(const std::string& progname) { - fmt::print("{} {}\n", progname, cargo::version_string); -} - -void -print_help(const std::string& progname, - const bpo::options_description& opt_desc) { - fmt::print("Usage: {} [options]\n\n", progname); - fmt::print("{}", opt_desc); -} - -std::unordered_map -load_environment_variables() { - - std::unordered_map envs; - - if(const auto p = std::getenv(cargo::env::LOG); - p && !std::string{p}.empty() && std::string{p} != "0") { - - if(const auto log_file = std::getenv(cargo::env::LOG_OUTPUT)) { - envs.emplace(cargo::env::LOG_OUTPUT, log_file); - } - } - - return envs; -} - -bpo::options_description -setup_command_line_args(config::settings& cfg) { - - - // define the command line options allowed - bpo::options_description opt_desc("Options"); - opt_desc.add_options() - // force logging messages to the console - ("force-console,C", - bpo::value() - ->implicit_value("") - ->zero_tokens() - ->notifier([&](const std::string&) { - cfg.log_file(fs::path{}); - cfg.use_console(true); - }), - "override any logging options defined in configuration files and " - "send all daemon output to the console") - - // print the daemon version - ("version,v", - bpo::value()->implicit_value("")->zero_tokens(), - "print version string") - - // print help - ("help,h", - bpo::value()->implicit_value("")->zero_tokens(), - "produce help message"); - - return opt_desc; -} +struct cargo_config { + std::string progname; + bool daemonize = false; + std::optional output_file; + std::string address; +}; -config::settings +cargo_config parse_command_line(int argc, char* argv[]) { - config::settings cfg; - cfg.daemonize(false); - - const auto opt_desc = ::setup_command_line_args(cfg); - - // parse the command line - bpo::variables_map vm; - - try { - bpo::store(bpo::parse_command_line(argc, argv, opt_desc), vm); - - // the --help and --version arguments are special, since we want - // to process them even if the global configuration file doesn't exist - if(vm.count("help")) { - print_help(cfg.progname(), opt_desc); - exit(EXIT_SUCCESS); - } - - if(vm.count("version")) { - print_version(cfg.progname()); - exit(EXIT_SUCCESS); - } + cargo_config cfg; - const fs::path config_file = (vm.count("config-file") == 0) - ? cfg.config_file() - : vm["config-file"].as(); + cfg.progname = fs::path{argv[0]}.filename().string(); - if(!fs::exists(config_file)) { - fmt::print(stderr, - "Failed to access daemon configuration file {}\n", - config_file); - exit(EXIT_FAILURE); - } + CLI::App app{"Cargo: A parallel data staging framework for HPC", + cfg.progname}; - try { - cfg.load_from_file(config_file); - } catch(const std::exception& ex) { - fmt::print(stderr, - "Failed reading daemon configuration file:\n" - " {}\n", - ex.what()); - exit(EXIT_FAILURE); - } + // force logging messages to file + app.add_option("-o,--output", cfg.output_file, + "Write any output to FILENAME rather than sending it to the " + "console") + ->option_text("FILENAME"); - // override settings from the configuration file with settings - // from environment variables - const auto env_opts = load_environment_variables(); + app.add_option("-l,--listen", cfg.address, + "Address or interface to bind the daemon to. If using " + "`libfabric`,\n" + "the address is typically in the form of:\n\n" + " ofi+[://:]\n\n" + "Check `fi_info` to see the list of available protocols.\n") + ->option_text("ADDRESS") + ->required(); - if(const auto& it = env_opts.find(cargo::env::LOG_OUTPUT); - it != env_opts.end()) { - cfg.log_file(it->second); - } - - // calling notify() here basically invokes all define notifiers, thus - // overriding any configuration loaded from the global configuration - // file with its command-line counterparts if provided (for those - // options where this is available) - bpo::notify(vm); + app.add_flag_function( + "-v,--version", + [&](auto /*count*/) { + fmt::print("{} {}\n", cfg.progname, cargo::version_string); + std::exit(EXIT_SUCCESS); + }, + "Print version and exit"); + try { + app.parse(argc, argv); return cfg; - } catch(const bpo::error& ex) { - fmt::print(stderr, "ERROR: {}\n\n", ex.what()); - exit(EXIT_FAILURE); + } catch(const CLI::ParseError& ex) { + std::exit(app.exit(ex)); } } @@ -180,7 +97,7 @@ parse_command_line(int argc, char* argv[]) { int main(int argc, char* argv[]) { - config::settings cfg = parse_command_line(argc, argv); + cargo_config cfg = parse_command_line(argc, argv); // Initialize the MPI environment mpi::environment env; @@ -196,7 +113,7 @@ main(int argc, char* argv[]) { fmt::print(stderr, "An unhandled exception reached the top of main(), " "{} will exit:\n what(): {}\n", - cfg.progname(), ex.what()); + cfg.progname, ex.what()); return EXIT_FAILURE; } diff --git a/src/master.cpp b/src/master.cpp index cb5eb0e..25abb23 100644 --- a/src/master.cpp +++ b/src/master.cpp @@ -23,7 +23,6 @@ *****************************************************************************/ #include -#include #include #include -- GitLab From 414fc9da890f800489281d2eceea97eda11ce614 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 31 Aug 2023 12:11:17 +0200 Subject: [PATCH 5/6] Replace `master` function with `master_server` class RPC handlers are now member functions of `master_server` --- src/cargo.cpp | 12 ++++- src/master.cpp | 96 +++++++++++++++++------------------- src/master.hpp | 20 ++++++++ src/proto/rpc/response.hpp | 99 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 53 deletions(-) create mode 100644 src/proto/rpc/response.hpp diff --git a/src/cargo.cpp b/src/cargo.cpp index e752637..889a1d5 100644 --- a/src/cargo.cpp +++ b/src/cargo.cpp @@ -105,7 +105,17 @@ main(int argc, char* argv[]) { try { if(world.rank() == 0) { - master(cfg); + + master_server srv{cfg.progname, cfg.address, cfg.daemonize, + fs::current_path()}; + + if(cfg.output_file) { + srv.configure_logger(logger::logger_type::file, + *cfg.output_file); + } + + return srv.run(); + } else { worker(); } diff --git a/src/master.cpp b/src/master.cpp index 25abb23..cca9704 100644 --- a/src/master.cpp +++ b/src/master.cpp @@ -29,17 +29,17 @@ #include #include #include +#include #include "message.hpp" +#include "master.hpp" +#include "net/utilities.hpp" +#include "net/request.hpp" +#include "proto/rpc/response.hpp" using namespace std::literals; namespace { -std::string -get_address(auto&& req) { - return req.get_endpoint(); -} - cargo::transfer_request_message create_request_message(const cargo::dataset& input, const cargo::dataset& output) { @@ -62,48 +62,55 @@ create_request_message(const cargo::dataset& input, using namespace std::literals; -struct remote_procedure { - static std::uint64_t - new_id() { - static std::atomic_uint64_t current_id; - return current_id++; - } -}; +master_server::master_server(std::string name, std::string address, + bool daemonize, std::filesystem::path rundir, + std::optional pidfile) + : server(std::move(name), std::move(address), daemonize, std::move(rundir), + std::move(pidfile)), + provider(m_network_engine, 0) { + +#define EXPAND(rpc_name) #rpc_name##s, &master_server::rpc_name + provider::define(EXPAND(ping)); + provider::define(EXPAND(transfer_datasets)); -namespace handlers { +#undef EXPAND +} + +#define RPC_NAME() ("ADM_"s + __FUNCTION__) void -ping(const thallium::request& req) { +master_server::ping(const network::request& req) { - const auto rpc_id = remote_procedure::new_id(); + using network::get_address; + using network::rpc_info; + using proto::generic_response; - LOGGER_INFO("rpc id: {} name: {} from: {} => " - "body: {{}}", - rpc_id, std::quoted(__FUNCTION__), - std::quoted(get_address(req))); + const auto rpc = rpc_info::create(RPC_NAME(), get_address(req)); - const auto retval = cargo::error_code{0}; + LOGGER_INFO("rpc {:>} body: {{}}", rpc); - LOGGER_INFO("rpc id: {} name: {} to: {} <= " - "body: {{retval: {}}}", - rpc_id, std::quoted(__FUNCTION__), - std::quoted(get_address(req)), retval); + const auto resp = generic_response{rpc.id(), cargo::error_code{0}}; - req.respond(retval); + LOGGER_INFO("rpc {:<} body: {{retval: {}}}", rpc, resp.error_code()); + + req.respond(resp); } void -transfer_datasets(const network::request& req, - const std::vector& sources, - const std::vector& targets) { +master_server::transfer_datasets(const network::request& req, + const std::vector& sources, + const std::vector& targets) { + + using network::get_address; + using network::rpc_info; + using proto::generic_response; - const auto rpc_id = remote_procedure::new_id(); - LOGGER_INFO("rpc id: {} name: {} from: {} => " - "body: {{sources: {}, targets: {}}}", - rpc_id, std::quoted(__FUNCTION__), - std::quoted(get_address(req)), sources, targets); + const auto rpc = rpc_info::create(RPC_NAME(), get_address(req)); - const auto retval = cargo::error_code{0}; + LOGGER_INFO("rpc {:>} body: {{sources: {}, targets: {}}}", rpc, sources, + targets); + + const auto resp = generic_response{rpc.id(), cargo::error_code{0}}; assert(sources.size() == targets.size()); @@ -123,23 +130,8 @@ transfer_datasets(const network::request& req, cargo::transfer tx{42}; - LOGGER_INFO("rpc id: {} name: {} to: {} <= " - "body: {{retval: {}, transfer: {}}}", - rpc_id, std::quoted(__FUNCTION__), - std::quoted(get_address(req)), retval, tx); - - req.respond(retval); -} - - -} // namespace handlers - -void -master(const config::settings& cfg) { - - network::server daemon(cfg); + LOGGER_INFO("rpc {:<} body: {{retval: {}, transfer: {}}}", rpc, + resp.error_code(), tx); - daemon.set_handler("ping"s, handlers::ping); - daemon.set_handler("transfer_datasets"s, handlers::transfer_datasets); - daemon.run(); + req.respond(resp); } diff --git a/src/master.hpp b/src/master.hpp index 31a9b09..93acbae 100644 --- a/src/master.hpp +++ b/src/master.hpp @@ -25,6 +25,26 @@ #ifndef CARGO_MASTER_HPP #define CARGO_MASTER_HPP +#include "net/server.hpp" +#include "cargo.hpp" + +class master_server : public network::server, + public network::provider { +public: + master_server(std::string name, std::string address, bool daemonize, + std::filesystem::path rundir, + std::optional pidfile = {}); + +private: + void + ping(const network::request& req); + + void + transfer_datasets(const network::request& req, + const std::vector& sources, + const std::vector& targets); +}; + namespace config { struct settings; } // namespace config diff --git a/src/proto/rpc/response.hpp b/src/proto/rpc/response.hpp new file mode 100644 index 0000000..2cc833b --- /dev/null +++ b/src/proto/rpc/response.hpp @@ -0,0 +1,99 @@ +/****************************************************************************** + * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of Cargo. + * + * Cargo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Cargo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Cargo. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#ifndef CARGO_PROTO_RPC_RESPONSE_HPP +#define CARGO_PROTO_RPC_RESPONSE_HPP + +#include +#include + +namespace proto { + +template +class generic_response { + +public: + constexpr generic_response() noexcept = default; + + constexpr generic_response(std::uint64_t op_id, Error&& ec) noexcept + : m_op_id(op_id), m_error_code(ec) {} + + [[nodiscard]] constexpr std::uint64_t + op_id() const noexcept { + return m_op_id; + } + + [[nodiscard]] constexpr Error + error_code() const noexcept { + return m_error_code; + } + + template + constexpr void + serialize(Archive&& ar) { + ar & m_op_id; + ar & m_error_code; + } + +private: + std::uint64_t m_op_id = 0; + Error m_error_code{}; +}; + +template +class response_with_value : public generic_response { + +public: + constexpr response_with_value() noexcept = default; + + constexpr response_with_value(std::uint64_t op_id, Error&& ec, + std::optional value) noexcept + : generic_response(op_id, ec), m_value(std::move(value)) {} + + constexpr auto + value() const noexcept { + return m_value.value(); + } + + constexpr auto + has_value() const noexcept { + return m_value.has_value(); + } + + template + constexpr void + serialize(Archive&& ar) { + ar(cereal::base_class>(this), m_value); + } + +private: + std::optional m_value; +}; + +template +using response_with_id = response_with_value; + +} // namespace proto + +#endif // CARGO_PROTO_RPC_RESPONSE_HPP -- GitLab From 2f4d30841a4ea851a5ebc4dff67d8789a18c7dee Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 1 Sep 2023 08:09:08 +0200 Subject: [PATCH 6/6] Remove `utils` module --- src/CMakeLists.txt | 1 - src/utils/CMakeLists.txt | 31 ----- src/utils/utils/CMakeLists.txt | 31 ----- src/utils/utils/ctype_ptr.hpp | 123 ------------------ src/utils/utils/signal_listener.hpp | 110 ---------------- src/utils/utils/utils.cpp | 191 ---------------------------- src/utils/utils/utils.hpp | 86 ------------- 7 files changed, 573 deletions(-) delete mode 100644 src/utils/CMakeLists.txt delete mode 100644 src/utils/utils/CMakeLists.txt delete mode 100644 src/utils/utils/ctype_ptr.hpp delete mode 100644 src/utils/utils/signal_listener.hpp delete mode 100644 src/utils/utils/utils.cpp delete mode 100644 src/utils/utils/utils.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c3538e7..ca9ff8c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -22,7 +22,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ -add_subdirectory(utils) add_subdirectory(logger) add_subdirectory(net) add_subdirectory(posix_file) diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt deleted file mode 100644 index b1c2966..0000000 --- a/src/utils/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -################################################################################ -# Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain # -# # -# This software was partially supported by the EuroHPC-funded project ADMIRE # -# (Project ID: 956748, https://www.admire-eurohpc.eu). # -# # -# This file is part of Cargo. # -# # -# Cargo is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# Cargo is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with Cargo. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ - -add_subdirectory(utils) - -target_include_directories( - utils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} -) - -set_property(TARGET utils PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/src/utils/utils/CMakeLists.txt b/src/utils/utils/CMakeLists.txt deleted file mode 100644 index aec0f16..0000000 --- a/src/utils/utils/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -################################################################################ -# Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain # -# # -# This software was partially supported by the EuroHPC-funded project ADMIRE # -# (Project ID: 956748, https://www.admire-eurohpc.eu). # -# # -# This file is part of Cargo. # -# # -# Cargo is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# Cargo is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with Cargo. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ - -add_library(utils STATIC) - -set_property(TARGET utils PROPERTY POSITION_INDEPENDENT_CODE ON) - -target_sources( - utils PRIVATE utils.hpp utils.cpp signal_listener.hpp ctype_ptr.hpp -) diff --git a/src/utils/utils/ctype_ptr.hpp b/src/utils/utils/ctype_ptr.hpp deleted file mode 100644 index c158809..0000000 --- a/src/utils/utils/ctype_ptr.hpp +++ /dev/null @@ -1,123 +0,0 @@ -/****************************************************************************** - * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of Cargo. - * - * Cargo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Cargo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Cargo. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#ifndef UTILS_C_PTR_HPP -#define UTILS_C_PTR_HPP - -#include -#include - -namespace utils { - -// A manager for a C raw pointer. It allows to automatically delete dynamically -// allocated C structs in a RAII manner (provided that there is a deleter -// function for the struct available). -template -struct deleter { - void - operator()(T* ptr) { - fn(ptr); - } -}; - -template -using ctype_ptr = - std::unique_ptr::type, - deleter::type, fn>>; - -// A manager for a vector of C raw pointers. It allows to automatically -// delete the dynamically allocated C structs pointed by each vector elements -// in a RAII manner (provided that there is a deleter function for the struct -// available). Can also be used to directly pass an array of C pointers to C -// APIs by means of the data() function. -template -struct ctype_ptr_vector { - - ctype_ptr_vector() = default; - - ctype_ptr_vector(T* const data, size_t size) { - reserve(size); - - for(size_t i = 0; i < size; ++i) { - emplace_back(data[i]); - } - } - - ctype_ptr_vector(ctype_ptr_vector&& rhs) noexcept - : m_data(std::move(rhs.m_data)), m_addrs(std::move(rhs.m_addrs)) {} - - ctype_ptr_vector& - operator=(ctype_ptr_vector&&) noexcept = default; - - ~ctype_ptr_vector() = default; - - constexpr void - reserve(size_t n) { - m_data.reserve(n); - m_addrs.reserve(n); - } - - template - constexpr void - emplace_back(Args&&... args) { - const auto& tmp = m_data.emplace_back(args...); - m_addrs.push_back(tmp.get()); - } - - constexpr const T* - data() const noexcept { - return m_addrs.data(); - } - - constexpr T* - data() noexcept { - return m_addrs.data(); - } - - constexpr std::size_t - size() const noexcept { - return m_data.size(); - } - - constexpr T* - release() noexcept { - - auto* data = (T*) calloc(m_data.size(), sizeof(T)); - - for(size_t i = 0; i < m_data.size(); ++i) { - data[i] = m_data[i].release(); - m_addrs[i] = nullptr; - } - - return data; - } - - - std::vector> m_data{}; - std::vector m_addrs{}; -}; - -} // namespace utils - -#endif // UTILS_C_PTR_HPP diff --git a/src/utils/utils/signal_listener.hpp b/src/utils/utils/signal_listener.hpp deleted file mode 100644 index 9e7981f..0000000 --- a/src/utils/utils/signal_listener.hpp +++ /dev/null @@ -1,110 +0,0 @@ -/****************************************************************************** - * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of Cargo. - * - * Cargo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Cargo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Cargo. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#ifndef UTILS_SIGNAL_LISTENER_HPP -#define UTILS_SIGNAL_LISTENER_HPP - -#include -#include - -namespace { -template -void -do_for(F f, Args... args) { - [[maybe_unused]] int x[] = {(f(args), 0)...}; -} -} // namespace - -namespace utils { - -namespace ba = boost::asio; - -struct signal_listener { - - using SignalHandlerType = std::function; - - signal_listener() : m_ios(), m_signals(m_ios) {} - - template - void - set_handler(SignalHandlerType&& handler, Args... signums) { - - m_user_handler = std::forward(handler); - m_signals.clear(); - - ::do_for([&](int signum) { m_signals.add(signum); }, signums...); - } - - void - clear_handler() { - m_user_handler = nullptr; - m_signals.clear(); - } - - void - do_accept() { - - if(m_user_handler) { - m_signals.async_wait(std::bind( // NOLINT - &signal_listener::signal_handler, this, - std::placeholders::_1, std::placeholders::_2)); - } - } - - void - run() { - std::thread([&]() { - do_accept(); - m_ios.run(); - }).detach(); - } - - void - stop() { - m_ios.stop(); - } - -private: - void - signal_handler(boost::system::error_code ec, int signal_number) { - // a signal occurred, invoke installed handler - if(!ec) { - m_user_handler(signal_number); - } - - // reinstall handler - m_signals.async_wait( - std::bind(&signal_listener::signal_handler, this, // NOLINT - std::placeholders::_1, std::placeholders::_2)); - } - - - ba::io_service m_ios; - ba::signal_set m_signals; - SignalHandlerType m_user_handler; -}; - -} // namespace utils - -#endif // UTILS_SIGNAL_LISTENER_HPP diff --git a/src/utils/utils/utils.cpp b/src/utils/utils/utils.cpp deleted file mode 100644 index a0ace9e..0000000 --- a/src/utils/utils/utils.cpp +++ /dev/null @@ -1,191 +0,0 @@ -/****************************************************************************** - * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of Cargo. - * - * Cargo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Cargo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Cargo. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils.hpp" - -namespace utils { - -uint64_t -parse_size(const std::string& str) { - - constexpr const uint64_t B_FACTOR = 1; - constexpr const uint64_t KB_FACTOR = 1e3; - constexpr const uint64_t KiB_FACTOR = (1 << 10); - constexpr const uint64_t MB_FACTOR = 1e6; - constexpr const uint64_t MiB_FACTOR = (1 << 20); - constexpr const uint64_t GB_FACTOR = 1e9; - constexpr const uint64_t GiB_FACTOR = (1 << 30); - - const std::pair conversions[] = { - {"GiB", GiB_FACTOR}, {"GB", GB_FACTOR}, {"G", GB_FACTOR}, - {"MiB", MiB_FACTOR}, {"MB", MB_FACTOR}, {"M", MB_FACTOR}, - {"KiB", KiB_FACTOR}, {"KB", KB_FACTOR}, {"K", KB_FACTOR}, - {"B", B_FACTOR}, - }; - - std::string scopy(str); - - /* remove whitespaces from the string */ - scopy.erase(std::remove_if(scopy.begin(), scopy.end(), - [](char ch) { - return std::isspace( - ch, std::locale::classic()); - }), - scopy.end()); - - /* determine the units */ - std::size_t pos = std::string::npos; - uint64_t factor = B_FACTOR; - - for(const auto& c : conversions) { - const std::string& suffix = c.first; - - if((pos = scopy.find(suffix)) != std::string::npos) { - /* check that the candidate is using the suffix EXACTLY - * to prevent accepting something like "GBfoo" as a valid Gigabyte - * unit */ - if(suffix != scopy.substr(pos)) { - pos = std::string::npos; - continue; - } - - factor = c.second; - break; - } - } - - /* this works as expected because if pos == std::string::npos, the standard - * states that all characters until the end of the string should be - * included.*/ - std::string number_str = scopy.substr(0, pos); - - /* check if it's a valid number */ - if(number_str.empty() || - !std::all_of(number_str.begin(), number_str.end(), ::isdigit)) { - throw std::invalid_argument("Not a number"); - } - - double value = std::stod(number_str); - - return std::round(value * factor); -} - -// lexically remove any ./ and ../ components from a provided pathname -// (adapted from boost::filesystem::path::lexically_normal()) -std::filesystem::path -lexical_normalize(const std::filesystem::path& pathname, bool as_directory) { - - using std::filesystem::path; - - if(pathname.empty()) { - return path{}; - } - - if(pathname == "/") { - return path{"/"}; - } - - path tmp{"/"}; - - for(const auto& elem : pathname) { - if(elem == "..") { - // move back on '../' - tmp = tmp.remove_filename(); - - if(tmp.empty()) { - tmp = "/"; - } - } else if(elem != "." && elem != "/") { - // There is a weird case in some versions of boost, where the elem - // is returned incorrectly for paths with 2 leading slashes: - // Example: *path('//a/b/c').begin() returns '//a' instead of '/' - // If this happens, we extract the trailing component - if(elem.native().size() > 2 && - ((elem.native())[0] == '/' && (elem.native())[1] == '/')) { - tmp /= elem.native().substr(2); - } else { - tmp /= elem; - } - } - // ignore './' - } - - if(tmp != "/" && as_directory) { - tmp /= "/"; - } - - return tmp; -} - - -// remove a trailing separator -// (adapted from boost::filesystem::path::remove_trailing_separator()) -std::filesystem::path -remove_trailing_separator(const std::filesystem::path& pathname) { - - using std::filesystem::path; - - std::string str{pathname.generic_string()}; - - auto is_directory_separator = [](path::value_type c) { - return c == '/'; - }; - - if(!str.empty() && is_directory_separator(str[str.size() - 1])) { - str.erase(str.size() - 1); - } - - return path{str}; -} - -// remove a leading separator -// (adapted from boost::filesystem::path::remove_trailing_separator()) -std::filesystem::path -remove_leading_separator(const std::filesystem::path& pathname) { - - using std::filesystem::path; - - std::string str{pathname.generic_string()}; - - auto is_directory_separator = [](path::value_type c) { - return c == '/'; - }; - - if(!str.empty() && is_directory_separator(str[0])) { - str.erase(0, 1); - } - - return path{str}; -} - -} // namespace utils diff --git a/src/utils/utils/utils.hpp b/src/utils/utils/utils.hpp deleted file mode 100644 index f387aa5..0000000 --- a/src/utils/utils/utils.hpp +++ /dev/null @@ -1,86 +0,0 @@ -/****************************************************************************** - * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of Cargo. - * - * Cargo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Cargo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Cargo. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#ifndef UTILS_HPP -#define UTILS_HPP - -#include -#include -#include -#include -#include - -namespace utils { - -template -class singleton { -public: - static T& - instance() { - static T s_instance; - return s_instance; - }; - - singleton(const singleton&) = delete; - singleton(singleton&&) = delete; - singleton& - operator=(const singleton&) = delete; - singleton& - operator=(singleton&&) = delete; - -protected: - struct token {}; - singleton() = default; - ~singleton() = default; -}; - -uint64_t -parse_size(const std::string& str); - -template -std::string -n2hexstr(T i, bool zero_pad = false) { - std::stringstream ss; - - if(zero_pad) { - ss << std::setfill('0') << std::setw(sizeof(T) << 1); - } - - ss << std::showbase << std::hex << i; - return ss.str(); -} - -std::filesystem::path -lexical_normalize(const std::filesystem::path& pathname, - bool as_directory = false); - -std::filesystem::path -remove_trailing_separator(const std::filesystem::path& pathname); - -std::filesystem::path -remove_leading_separator(const std::filesystem::path& pathname); - -} // namespace utils - -#endif // UTILS_HPP -- GitLab