Merge branch... (044059fb) · Commits · hpc / cargo

CMakeLists.txt

+41 −31

Original line number	Diff line number	Diff line
		@@ -4,20 +4,20 @@
		# This software was partially supported by the EuroHPC-funded project ADMIRE #
		# (Project ID: 956748, https://www.admire-eurohpc.eu). #
		# #
		# This file is part of cargo. #
		# This file is part of Cargo. #
		# #
		# cargo is free software: you can redistribute it and/or modify #
		# Cargo is free software: you can redistribute it and/or modify #
		# it under the terms of the GNU General Public License as published by #
		# the Free Software Foundation, either version 3 of the License, or #
		# (at your option) any later version. #
		# #
		# cargo is distributed in the hope that it will be useful, #
		# Cargo is distributed in the hope that it will be useful, #
		# but WITHOUT ANY WARRANTY; without even the implied warranty of #
		# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
		# GNU General Public License for more details. #
		# #
		# You should have received a copy of the GNU General Public License #
		# along with cargo. If not, see <https://www.gnu.org/licenses/>. #
		# along with Cargo. If not, see <https://www.gnu.org/licenses/>. #
		# #
		# SPDX-License-Identifier: GPL-3.0-or-later #
		################################################################################
		@@ -159,9 +159,9 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
		### instead, make sure that pkg-config is available
		find_package(PkgConfig REQUIRED)

		### boost libraries: required for processing program options
		### boost libraries: required for boost::mpi
		message(STATUS "[${PROJECT_NAME}] Checking for boost libraries")
		find_package(Boost 1.53 REQUIRED COMPONENTS program_options mpi)
		find_package(Boost 1.53 REQUIRED COMPONENTS mpi)

		### transport library
		if (CARGO_TRANSPORT_LIBRARY STREQUAL libfabric)
		@@ -216,35 +216,18 @@ FetchContent_Declare(
		FetchContent_MakeAvailable(spdlog)
		set_target_properties(spdlog PROPERTIES POSITION_INDEPENDENT_CODE ON)

		### file_options: required for reading configuration files
		message(STATUS "[${PROJECT_NAME}] Downloading and building file_options")
		FetchContent_Declare(
		file_options
		GIT_REPOSITORY https://storage.bsc.es/gitlab/utils/file_options
		GIT_TAG bdb4f7f7f2dd731815241fc41afe6373df8f732a # v0.1.0-pre
		### CLI11: used for parsing command-line options
		message(STATUS "[${PROJECT_NAME}] Searching for CLI11")
		FetchContent_Declare(cli11
		GIT_REPOSITORY https://github.com/CLIUtils/CLI11
		GIT_TAG 291c58789c031208f08f4f261a858b5b7083e8e2 # v2.3.2
		GIT_SHALLOW ON
		GIT_PROGRESS ON
		)

		FetchContent_MakeAvailable(file_options)

		### genopts: required for generating file_options schemas
		message(STATUS "[${PROJECT_NAME}] Downloading and building genopts")
		FetchContent_Declare(
		genopts
		GIT_REPOSITORY https://storage.bsc.es/gitlab/utils/genopts
		GIT_TAG c456c2d8ec92f26d9074b123446261103e5c847c # v0.1.0-pre
		# enabling GIT_SHALLOW when the GIT_TAG value is not backed by an
		# actual tag is problematic
		GIT_SHALLOW OFF
		GIT_PROGRESS ON
		)

		FetchContent_MakeAvailable(genopts)
		FetchContent_MakeAvailable(cli11)

		### expected: required for using tl::expected in the C++ library implementation
		### until std::expected makes it to C++

		message(STATUS "[${PROJECT_NAME}] Downloading and building tl::expected")
		set(EXPECTED_BUILD_PACKAGE OFF)
		set(EXPECTED_BUILD_TESTS OFF)
		@@ -258,6 +241,33 @@ FetchContent_Declare(

		FetchContent_MakeAvailable(expected)

		### Threads: required by ASIO
		find_package(Threads REQUIRED)

		### ASIO: used for signal handling
		###
		### ASIO is based on autotools and not CMake package. We can use
		### FetchContent to download it but we need to manually define the imported
		### target ourselves rather than relying on FetchContent_MakeAvailable()
		### to do it.
		message(STATUS "[${PROJECT_NAME}] Searching for ASIO")
		FetchContent_Declare(
		asio
		GIT_REPOSITORY https://github.com/chriskohlhoff/asio.git
		GIT_TAG asio-1-18-2
		CONFIGURE_COMMAND "" BUILD_COMMAND ""
		)
		FetchContent_GetProperties(asio)
		if(NOT asio_POPULATED)
		FetchContent_Populate(asio)
		endif()

		add_library(asio INTERFACE)
		target_include_directories(asio INTERFACE ${asio_SOURCE_DIR}/asio/include)
		target_link_libraries(asio INTERFACE Threads::Threads)
		add_library(asio::asio ALIAS asio)


		pkg_check_modules(LIBCONFIG IMPORTED_TARGET libconfig>=1.4.9)

		if (CARGO_BUILD_TESTS)

lib/CMakeLists.txt

+1 −1

Original line number	Diff line number	Diff line
		@@ -36,7 +36,7 @@ target_include_directories(

		set_target_properties(cargo PROPERTIES PUBLIC_HEADER "${public_headers}")

		target_link_libraries(cargo PRIVATE logger fmt::fmt thallium)
		target_link_libraries(cargo PRIVATE logger::logger fmt::fmt thallium)

		## Install library + targets ###################################################

src/CMakeLists.txt

+1 −4

Original line number	Diff line number	Diff line
		@@ -22,8 +22,6 @@
		# SPDX-License-Identifier: GPL-3.0-or-later #
		################################################################################

		add_subdirectory(utils)
		add_subdirectory(config)
		add_subdirectory(logger)
		add_subdirectory(net)
		add_subdirectory(posix_file)
		@@ -52,12 +50,11 @@ target_include_directories(
		target_link_libraries(
		cargo_server
		PRIVATE config
		logger
		rpc_server
		cargo
		fmt::fmt
		MPI::MPI_CXX
		Boost::program_options
		CLI11::CLI11
		Boost::serialization
		Boost::mpi
		posix_file

src/cargo.cpp

+50 −123

Original line number	Diff line number	Diff line
		@@ -24,154 +24,71 @@


		#include <filesystem>
		#include <boost/program_options.hpp>
		#include <fmt/format.h>
		#include <fmt/ostream.h>
		#include <exception>
		#include <cstdio>
		#include <cstdlib>
		#include <string>
		#include <CLI/CLI.hpp>
		#include <boost/mpi.hpp>

		#include <version.hpp>
		#include <config/settings.hpp>
		#include <boost/mpi.hpp>
		#include "master.hpp"
		#include "worker.hpp"
		#include "env.hpp"

		namespace fs = std::filesystem;
		namespace bpo = boost::program_options;
		namespace mpi = boost::mpi;

		// utility functions
		namespace {

		void
		print_version(const std::string& progname) {
		fmt::print("{} {}\n", progname, cargo::version_string);
		}

		void
		print_help(const std::string& progname,
		const bpo::options_description& opt_desc) {
		fmt::print("Usage: {} [options]\n\n", progname);
		fmt::print("{}", opt_desc);
		}

		std::unordered_map<std::string, std::string>
		load_environment_variables() {

		std::unordered_map<std::string, std::string> envs;

		if(const auto p = std::getenv(cargo::env::LOG);
		p && !std::string{p}.empty() && std::string{p} != "0") {

		if(const auto log_file = std::getenv(cargo::env::LOG_OUTPUT)) {
		envs.emplace(cargo::env::LOG_OUTPUT, log_file);
		}
		}
		struct cargo_config {
		std::string progname;
		bool daemonize = false;
		std::optional<fs::path> output_file;
		std::string address;
		};

		return envs;
		}

		bpo::options_description
		setup_command_line_args(config::settings& cfg) {


		// define the command line options allowed
		bpo::options_description opt_desc("Options");
		opt_desc.add_options()
		// force logging messages to the console
		("force-console,C",
		bpo::value<std::string>()
		->implicit_value("")
		->zero_tokens()
		->notifier([&](const std::string&) {
		cfg.log_file(fs::path{});
		cfg.use_console(true);
		}),
		"override any logging options defined in configuration files and "
		"send all daemon output to the console")

		// print the daemon version
		("version,v",
		bpo::value<std::string>()->implicit_value("")->zero_tokens(),
		"print version string")

		// print help
		("help,h",
		bpo::value<std::string>()->implicit_value("")->zero_tokens(),
		"produce help message");

		return opt_desc;
		}

		config::settings
		cargo_config
		parse_command_line(int argc, char* argv[]) {

		config::settings cfg;
		cfg.daemonize(false);
		cargo_config cfg;

		const auto opt_desc = ::setup_command_line_args(cfg);
		cfg.progname = fs::path{argv[0]}.filename().string();

		// parse the command line
		bpo::variables_map vm;
		CLI::App app{"Cargo: A parallel data staging framework for HPC",
		cfg.progname};

		try {
		bpo::store(bpo::parse_command_line(argc, argv, opt_desc), vm);
		// force logging messages to file
		app.add_option("-o,--output", cfg.output_file,
		"Write any output to FILENAME rather than sending it to the "
		"console")
		->option_text("FILENAME");

		// the --help and --version arguments are special, since we want
		// to process them even if the global configuration file doesn't exist
		if(vm.count("help")) {
		print_help(cfg.progname(), opt_desc);
		exit(EXIT_SUCCESS);
		}
		app.add_option("-l,--listen", cfg.address,
		"Address or interface to bind the daemon to. If using "
		"`libfabric`,\n"
		"the address is typically in the form of:\n\n"
		" ofi+<protocol>[://<hostname,IP,interface>:<port>]\n\n"
		"Check `fi_info` to see the list of available protocols.\n")
		->option_text("ADDRESS")
		->required();

		if(vm.count("version")) {
		print_version(cfg.progname());
		exit(EXIT_SUCCESS);
		}

		const fs::path config_file = (vm.count("config-file") == 0)
		? cfg.config_file()
		: vm["config-file"].as<fs::path>();

		if(!fs::exists(config_file)) {
		fmt::print(stderr,
		"Failed to access daemon configuration file {}\n",
		config_file);
		exit(EXIT_FAILURE);
		}
		app.add_flag_function(
		"-v,--version",
		[&](auto /count/) {
		fmt::print("{} {}\n", cfg.progname, cargo::version_string);
		std::exit(EXIT_SUCCESS);
		},
		"Print version and exit");

		try {
		cfg.load_from_file(config_file);
		} catch(const std::exception& ex) {
		fmt::print(stderr,
		"Failed reading daemon configuration file:\n"
		" {}\n",
		ex.what());
		exit(EXIT_FAILURE);
		}

		// override settings from the configuration file with settings
		// from environment variables
		const auto env_opts = load_environment_variables();

		if(const auto& it = env_opts.find(cargo::env::LOG_OUTPUT);
		it != env_opts.end()) {
		cfg.log_file(it->second);
		}

		// calling notify() here basically invokes all define notifiers, thus
		// overriding any configuration loaded from the global configuration
		// file with its command-line counterparts if provided (for those
		// options where this is available)
		bpo::notify(vm);

		app.parse(argc, argv);
		return cfg;
		} catch(const bpo::error& ex) {
		fmt::print(stderr, "ERROR: {}\n\n", ex.what());
		exit(EXIT_FAILURE);
		} catch(const CLI::ParseError& ex) {
		std::exit(app.exit(ex));
		}
		}

		@@ -180,7 +97,7 @@ parse_command_line(int argc, char* argv[]) {
		int
		main(int argc, char* argv[]) {

		config::settings cfg = parse_command_line(argc, argv);
		cargo_config cfg = parse_command_line(argc, argv);

		// Initialize the MPI environment
		mpi::environment env;
		@@ -188,7 +105,17 @@ main(int argc, char* argv[]) {

		try {
		if(world.rank() == 0) {
		master(cfg);

		master_server srv{cfg.progname, cfg.address, cfg.daemonize,
		fs::current_path()};

		if(cfg.output_file) {
		srv.configure_logger(logger::logger_type::file,
		*cfg.output_file);
		}

		return srv.run();

		} else {
		worker();
		}
		@@ -196,7 +123,7 @@ main(int argc, char* argv[]) {
		fmt::print(stderr,
		"An unhandled exception reached the top of main(), "
		"{} will exit:\n what(): {}\n",
		cfg.progname(), ex.what());
		cfg.progname, ex.what());

		return EXIT_FAILURE;
		}

src/config/CMakeLists.txt

deleted100644 → 0

+0 −34

Original line number	Diff line number	Diff line
		################################################################################
		# Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain #
		# #
		# This software was partially supported by the EuroHPC-funded project ADMIRE #
		# (Project ID: 956748, https://www.admire-eurohpc.eu). #
		# #
		# This file is part of Cargo. #
		# #
		# Cargo is free software: you can redistribute it and/or modify #
		# it under the terms of the GNU General Public License as published by #
		# the Free Software Foundation, either version 3 of the License, or #
		# (at your option) any later version. #
		# #
		# Cargo is distributed in the hope that it will be useful, #
		# but WITHOUT ANY WARRANTY; without even the implied warranty of #
		# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
		# GNU General Public License for more details. #
		# #
		# You should have received a copy of the GNU General Public License #
		# along with Cargo. If not, see <https://www.gnu.org/licenses/>. #
		# #
		# SPDX-License-Identifier: GPL-3.0-or-later #
		################################################################################

		add_subdirectory(config)

		# Since some of the sources will be auto-generated, we need to search for
		# includes in ${CMAKE_CURRENT_BINARY_DIR}
		target_include_directories(
		config PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
		${CMAKE_CURRENT_BINARY_DIR}
		)

		set_property(TARGET config PROPERTY POSITION_INDEPENDENT_CODE ON)