From bcd3ebf2a77a7ae94cea8f4c1c76b374d92a55e8 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 20 May 2022 18:15:31 +0200 Subject: [PATCH 1/5] Use genopts to generate config file schemas --- CMakeLists.txt | 25 ++++++++++ src/config/CMakeLists.txt | 28 ++++++++--- src/config/config_schema.hpp | 90 ------------------------------------ src/config/keywords.hpp | 55 ---------------------- src/config/settings.cpp | 2 +- 5 files changed, 47 insertions(+), 153 deletions(-) delete mode 100644 src/config/config_schema.hpp delete mode 100644 src/config/keywords.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f461309..b5f5beef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -188,6 +188,31 @@ FetchContent_Declare( FetchContent_MakeAvailable(spdlog) +### file_options: required for reading configuration files +message(STATUS "[${PROJECT_NAME}] Downloading and building file_options") +FetchContent_Declare( + file_options + GIT_REPOSITORY https://storage.bsc.es/gitlab/utils/file_options + GIT_TAG b2de92b3755e3391595bb368573b82abed16978d # v0.1.0-pre + GIT_SHALLOW ON + GIT_PROGRESS ON +) + +FetchContent_MakeAvailable(file_options) + +### genopts: required for generating file_options schemas +message(STATUS "[${PROJECT_NAME}] Downloading and building genopts") +FetchContent_Declare( + genopts + GIT_REPOSITORY https://storage.bsc.es/gitlab/utils/genopts + GIT_TAG 1dcef400f8fbc6e1969c856ca844707b730c3002 # v0.1.0-pre + GIT_SHALLOW ON + GIT_PROGRESS ON +) + +FetchContent_MakeAvailable(genopts) + + ### Mark any CMake variables imported from {fmt} and spdlog as advanced, so ### that they don't appear in cmake-gui or ccmake. Similarly for FETCHCONTENT ### variables. diff --git a/src/config/CMakeLists.txt b/src/config/CMakeLists.txt index ca9281a5..58ed2c05 100644 --- a/src/config/CMakeLists.txt +++ b/src/config/CMakeLists.txt @@ -26,26 +26,40 @@ add_library(config STATIC) target_include_directories( config PRIVATE ${CMAKE_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} ) set_property(TARGET config PROPERTY POSITION_INDEPENDENT_CODE ON) configure_file(defaults.cpp.in defaults.cpp) +configure_file(genopts.yml.in genopts.yml @ONLY) + +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/config_options.hpp + ${CMAKE_CURRENT_BINARY_DIR}/keywords.hpp + COMMENT "Generating config_options.hpp, keywords.hpp" + DEPENDS file_options.yml ${CMAKE_CURRENT_BINARY_DIR}/genopts.yml + COMMAND + Genopts::Python3_Interpreter -m genopts --config + ${CMAKE_CURRENT_BINARY_DIR}/genopts.yml + ${CMAKE_CURRENT_LIST_DIR}/file_options.yml +) + target_sources( config PRIVATE config.hpp - config_schema.hpp defaults.hpp ${CMAKE_CURRENT_BINARY_DIR}/defaults.cpp - file_options/file_options.hpp - file_options/schema.hpp - file_options/options_description.hpp - file_options/yaml_parser.hpp - keywords.hpp + ${CMAKE_CURRENT_BINARY_DIR}/config_options.hpp + ${CMAKE_CURRENT_BINARY_DIR}/keywords.hpp + file_options/file_options.hpp + file_options/schema.hpp + file_options/options_description.hpp + file_options/yaml_parser.hpp parsers.cpp parsers.hpp settings.cpp settings.hpp ) -target_link_libraries(config PRIVATE utils YAMLCpp::YAMLCpp) +target_link_libraries(config PRIVATE utils file_options::file_options) diff --git a/src/config/config_schema.hpp b/src/config/config_schema.hpp deleted file mode 100644 index b1441672..00000000 --- a/src/config/config_schema.hpp +++ /dev/null @@ -1,90 +0,0 @@ -/****************************************************************************** - * Copyright 2021, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of scord. - * - * scord is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * scord is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with scord. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#ifndef SCORD_CONFIG_SCHEMA_HPP -#define SCORD_CONFIG_SCHEMA_HPP - -#include - -#include "file_options/file_options.hpp" -#include "parsers.hpp" -#include "keywords.hpp" -#include "defaults.hpp" - -namespace fs = std::filesystem; - -namespace scord::config { - -using file_options::converter; -using file_options::declare_file; -using file_options::declare_group; -using file_options::declare_list; -using file_options::declare_option; -using file_options::file_schema; -using file_options::opt_type; -using file_options::sec_type; - -// define the configuration file structure and declare the supported options -const file_schema valid_options = declare_file({ - // section for global settings - declare_section( - keywords::global_settings, sec_type::mandatory, - declare_group({ - - declare_option( - keywords::use_syslog, opt_type::mandatory, - converter(parsers::parse_bool)), - - declare_option( - keywords::log_file, opt_type::optional, - converter(parsers::parse_path)), - - declare_option( - keywords::log_file_max_size, opt_type::optional, - converter(parsers::parse_capacity)), - - declare_option( - keywords::transport_protocol, - opt_type::mandatory), - - declare_option(keywords::bind_address, - opt_type::mandatory), - - declare_option( - keywords::remote_port, opt_type::mandatory, - converter(parsers::parse_number)), - - declare_option( - keywords::pidfile, opt_type::mandatory, - converter(parsers::parse_path)), - - declare_option( - keywords::workers, opt_type::mandatory, - converter(parsers::parse_number)), - })), -}); - -} // namespace scord::config - -#endif /* SCORD_CONFIG_SCHEMA_HPP */ diff --git a/src/config/keywords.hpp b/src/config/keywords.hpp deleted file mode 100644 index 6864cfe7..00000000 --- a/src/config/keywords.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/****************************************************************************** - * Copyright 2021, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of scord. - * - * scord is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * scord is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with scord. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#ifndef SCORD_CONFIG_KEYWORDS_HPP -#define SCORD_CONFIG_KEYWORDS_HPP - -namespace scord::config::keywords { - -// section names -constexpr static const auto global_settings = "global_settings"; -constexpr static const auto namespaces = "namespaces"; - -// option names for 'global-settings' section -constexpr static const auto use_syslog = "use_syslog"; -constexpr static const auto log_file = "log_file"; -constexpr static const auto log_file_max_size = "log_file_max_size"; -constexpr static const auto transport_protocol = "transport_protocol"; -constexpr static const auto bind_address = "bind_address"; -constexpr static const auto remote_port = "remote_port"; -constexpr static const auto pidfile = "pidfile"; -constexpr static const auto workers = "workers"; -constexpr static const auto staging_directory = "staging_directory"; - -// option names for 'namespaces' section -constexpr static const auto nsid = "nsid"; -constexpr static const auto track_contents = "track_contents"; -constexpr static const auto mountpoint = "mountpoint"; -constexpr static const auto type = "type"; -constexpr static const auto capacity = "capacity"; -constexpr static const auto visibility = "visibility"; - -} // namespace scord::config::keywords - -#endif /* SCORD_CONFIG_KEYWORDS_HPP */ diff --git a/src/config/settings.cpp b/src/config/settings.cpp index 1e9af35a..0987446d 100644 --- a/src/config/settings.cpp +++ b/src/config/settings.cpp @@ -24,7 +24,7 @@ #include #include -#include "config_schema.hpp" +#include "config_options.hpp" #include "defaults.hpp" #include "file_options/options_description.hpp" #include "file_options/yaml_parser.hpp" -- GitLab From e01d4fb212992455b0f67419f9accafac175a546 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 20 May 2022 18:16:07 +0200 Subject: [PATCH 2/5] Reformat CMakeFiles --- CMakeLists.txt | 70 ++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b5f5beef..e9d0f676 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,21 +38,21 @@ project( # Set default build type and also populate a list of available options get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) -if(NOT is_multi_config) +if (NOT is_multi_config) set(default_build_type "Release") set(allowed_build_types Debug Release MinSizeRel RelWithDebInfo) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "${allowed_build_types}") - if(NOT CMAKE_BUILD_TYPE) + if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE - "${default_build_type}" - CACHE STRING "Choose the type of build." FORCE - ) - elseif(NOT CMAKE_BUILD_TYPE IN_LIST allowed_build_types) + "${default_build_type}" + CACHE STRING "Choose the type of build." FORCE + ) + elseif (NOT CMAKE_BUILD_TYPE IN_LIST allowed_build_types) message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}") - endif() -endif() + endif () +endif () # make sure that debug versions for targets are used (if provided) in Debug mode set_property(GLOBAL APPEND PROPERTY DEBUG_CONFIGURATIONS Debug) @@ -83,10 +83,10 @@ include(scord-utils) ### transport library set(SCORD_TRANSPORT_LIBRARY - "libfabric" - CACHE STRING - "Transport library used by ${PROJECT_NAME} (default: libfabric)" -) + "libfabric" + CACHE STRING + "Transport library used by ${PROJECT_NAME} (default: libfabric)" + ) set_property(CACHE SCORD_TRANSPORT_LIBRARY PROPERTY STRINGS libfabric ucx) message( STATUS "[${PROJECT_NAME}] Transport library: ${SCORD_TRANSPORT_LIBRARY}" @@ -94,31 +94,31 @@ message( ### server transport protocol set(SCORD_TRANSPORT_PROTOCOL - "tcp" - CACHE - STRING - "Change the default transport protocol for the ${PROJECT_NAME} server (default: tcp)" -) + "tcp" + CACHE + STRING + "Change the default transport protocol for the ${PROJECT_NAME} server (default: tcp)" + ) message( STATUS - "[${PROJECT_NAME}] server default transport protocol: ${SCORD_TRANSPORT_PROTOCOL}" + "[${PROJECT_NAME}] server default transport protocol: ${SCORD_TRANSPORT_PROTOCOL}" ) ### server bind address set(SCORD_BIND_ADDRESS - "127.0.0.1" - CACHE - STRING - "Define the bind address for the ${PROJECT_NAME} server (default: 127.0.0.1)" -) + "127.0.0.1" + CACHE + STRING + "Define the bind address for the ${PROJECT_NAME} server (default: 127.0.0.1)" + ) message(STATUS "[${PROJECT_NAME}] server bind address: ${SCORD_BIND_ADDRESS}") ### server bind port set(SCORD_BIND_PORT - "52000" - CACHE STRING - "Define the bind port for the ${PROJECT_NAME} server (default: 52000)" -) + "52000" + CACHE STRING + "Define the bind port for the ${PROJECT_NAME} server (default: 52000)" + ) message(STATUS "[${PROJECT_NAME}] server bind port: ${SCORD_BIND_PORT}") option(SCORD_BUILD_EXAMPLES "Build examples (disabled by default)" OFF) @@ -137,20 +137,16 @@ find_package(PkgConfig REQUIRED) message(STATUS "[${PROJECT_NAME}] Checking for boost libraries") find_package(Boost 1.53 REQUIRED COMPONENTS program_options) -### yaml-cpp: required for reading configuration files -message(STATUS "[${PROJECT_NAME}] Checking for yaml-cpp") -find_package(YAMLCpp 0.6.2 REQUIRED) - ### transport library -if(SCORD_TRANSPORT_LIBRARY STREQUAL libfabric) +if (SCORD_TRANSPORT_LIBRARY STREQUAL libfabric) pkg_check_modules(libfabric REQUIRED IMPORTED_TARGET GLOBAL libfabric) add_library(transport_library ALIAS PkgConfig::libfabric) -elseif(SCORD_TRANSPORT_LIBRARY STREQUAL ucx) +elseif (SCORD_TRANSPORT_LIBRARY STREQUAL ucx) pkg_check_modules(ucx REQUIRED IMPORTED_TARGET GLOBAL ucx) add_library(transport_library ALIAS PkgConfig::ucx) -else() +else () message(FATAL_ERROR "Unknown transport library: ${SCORD_TRANSPORT_LIBRARY}") -endif() +endif () ### Mercury message(STATUS "[${PROJECT_NAME}] Checking for Mercury") @@ -224,6 +220,6 @@ mark_variables_as_advanced(REGEX "^(FETCHCONTENT|fmt|FMT|spdlog|SPDLOG)_.*$") add_subdirectory(etc) add_subdirectory(src) -if(SCORD_BUILD_EXAMPLES) +if (SCORD_BUILD_EXAMPLES) add_subdirectory(examples) -endif() +endif () -- GitLab From fd863b4dfa862e592d4387e0ae93f13ddd3b11f4 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 20 May 2022 18:35:28 +0200 Subject: [PATCH 3/5] Use the file_options target instead of source files --- src/config/CMakeLists.txt | 60 ++- src/config/file_options.yml | 41 ++ src/config/file_options/file_options.hpp | 32 -- .../file_options/options_description.hpp | 123 ------ src/config/file_options/schema.hpp | 414 ------------------ src/config/file_options/yaml_parser.hpp | 244 ----------- src/config/genopts.yml.in | 14 + 7 files changed, 96 insertions(+), 832 deletions(-) create mode 100644 src/config/file_options.yml delete mode 100644 src/config/file_options/file_options.hpp delete mode 100644 src/config/file_options/options_description.hpp delete mode 100644 src/config/file_options/schema.hpp delete mode 100644 src/config/file_options/yaml_parser.hpp create mode 100644 src/config/genopts.yml.in diff --git a/src/config/CMakeLists.txt b/src/config/CMakeLists.txt index 58ed2c05..e1e08003 100644 --- a/src/config/CMakeLists.txt +++ b/src/config/CMakeLists.txt @@ -22,29 +22,17 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ +# Create a config target for all configuration code add_library(config STATIC) +# Since some of the sources will be auto-generated, we need to search for +# includes in ${CMAKE_CURRENT_BINARY_DIR} target_include_directories( config PRIVATE ${CMAKE_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ) set_property(TARGET config PROPERTY POSITION_INDEPENDENT_CODE ON) -configure_file(defaults.cpp.in defaults.cpp) - -configure_file(genopts.yml.in genopts.yml @ONLY) - -add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/config_options.hpp - ${CMAKE_CURRENT_BINARY_DIR}/keywords.hpp - COMMENT "Generating config_options.hpp, keywords.hpp" - DEPENDS file_options.yml ${CMAKE_CURRENT_BINARY_DIR}/genopts.yml - COMMAND - Genopts::Python3_Interpreter -m genopts --config - ${CMAKE_CURRENT_BINARY_DIR}/genopts.yml - ${CMAKE_CURRENT_LIST_DIR}/file_options.yml -) - target_sources( config PRIVATE config.hpp @@ -52,10 +40,6 @@ target_sources( ${CMAKE_CURRENT_BINARY_DIR}/defaults.cpp ${CMAKE_CURRENT_BINARY_DIR}/config_options.hpp ${CMAKE_CURRENT_BINARY_DIR}/keywords.hpp - file_options/file_options.hpp - file_options/schema.hpp - file_options/options_description.hpp - file_options/yaml_parser.hpp parsers.cpp parsers.hpp settings.cpp @@ -63,3 +47,41 @@ target_sources( ) target_link_libraries(config PRIVATE utils file_options::file_options) + +# ############################################################################## +# Produce several auto-generated files for 'config' +# ############################################################################## + +# Default values for options +configure_file(defaults.cpp.in defaults.cpp) + +# Automatic generation of file_options schemas. To facilitate the management of +# any file-based configuration options, we generate a C++ schema from a YAML +# file that allows the 'file_options' library to parse them fromm a +# configuration file. The process works as follows: +# 1. We define the desired options in 'file_options.yml' +# 2. We rely on the 'genopts' tool to generate valid C++ schemas for the +# 'file_options' library. This tool can be configured using a 'genopts.yml' +# configuration file. +# 3. In the daemon code, we use the facilities provided by the 'file_options' +# library to access the options values. + +# Since the configuration in genopts.yml can be path-dependant, we rely on +# CMake to substitute any special @variables@ for their actual values +configure_file(genopts.yml.in genopts.yml @ONLY) + +# Define the command that will generate config_options.hpp and keywords.hpp. +# It will be executed since there is a direct dependency between 'config' +# (defined below) and these output files. +# We also make the command depend on file_options.yml and genopts.yml so that +# it gets re-executed if they change. +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/config_options.hpp + ${CMAKE_CURRENT_BINARY_DIR}/keywords.hpp + COMMENT "Generating config_options.hpp, keywords.hpp" + DEPENDS file_options.yml ${CMAKE_CURRENT_BINARY_DIR}/genopts.yml + COMMAND + Genopts::Python3_Interpreter -m genopts --config + ${CMAKE_CURRENT_BINARY_DIR}/genopts.yml + ${CMAKE_CURRENT_LIST_DIR}/file_options.yml +) diff --git a/src/config/file_options.yml b/src/config/file_options.yml new file mode 100644 index 00000000..7049676e --- /dev/null +++ b/src/config/file_options.yml @@ -0,0 +1,41 @@ +sections: + - name: "global_settings" + required: true + options: + - name: "use_syslog" + required: true + type: "bool" + converter: "parsers::parse_bool" + + - name: "log_file" + required: false + type: "std::filesystem::path" + converter: "parsers::parse_path" + + - name: "log_file_max_size" + required: false + type: "uint32_t" + converter: "parsers::parse_capacity" + + - name: "transport_protocol" + required: true + type: "std::string" + + - name: "bind_address" + required: true + type: "std::string" + + - name: "remote_port" + required: true + type: "uint32_t" + converter: "parsers::parse_number" + + - name: "pidfile" + required: true + type: "std::filesystem::path" + converter: "parsers::parse_path" + + - name: "workers" + required: true + type: "uint32_t" + converter: "parsers::parse_number" diff --git a/src/config/file_options/file_options.hpp b/src/config/file_options/file_options.hpp deleted file mode 100644 index c4260d23..00000000 --- a/src/config/file_options/file_options.hpp +++ /dev/null @@ -1,32 +0,0 @@ -/****************************************************************************** - * Copyright 2021, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of scord. - * - * scord is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * scord is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with scord. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#ifndef SCORD_CONFIG_FILE_OPTIONS_HPP -#define SCORD_CONFIG_FILE_OPTIONS_HPP - -#include "schema.hpp" -#include "options_description.hpp" -#include "yaml_parser.hpp" - -#endif /* SCORD_CONFIG_FILE_OPTIONS_HPP */ diff --git a/src/config/file_options/options_description.hpp b/src/config/file_options/options_description.hpp deleted file mode 100644 index 643a1426..00000000 --- a/src/config/file_options/options_description.hpp +++ /dev/null @@ -1,123 +0,0 @@ -/****************************************************************************** - * Copyright 2021, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of scord. - * - * scord is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * scord is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with scord. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -/* - * This file contains the code to implement an options map that can - * be generated by parsing a configuration according to the structure - * defined by a file_schema instance. - */ - -#ifndef SCORD_CONFIG_FILE_OPTIONS_DESCRIPTION_HPP -#define SCORD_CONFIG_FILE_OPTIONS_DESCRIPTION_HPP - -#include -#include -#include - -namespace file_options { - -/*! Class to handle values of an arbitrary type */ -struct option_value { - /*! Constructor. The 'value' parameter is passed (and stored) - * as boost::any in order to handle arbitrarily-typed values */ - explicit option_value(boost::any value) : m_value(std::move(value)) {} - - /*! If the stored value is of type T, returns that value. - * Otherwise throws boost::bad_any_cast exception */ - template - T& - get_as() { - return boost::any_cast(m_value); - } - - /*! If the stored value is of type T, returns that value. - * Otherwise throws boost::bad_any_cast exception */ - template - const T& - get_as() const { - return boost::any_cast(m_value); - } - - /*! Stored option value */ - boost::any m_value; -}; - -/*! Class to handle a group of related options. The stored options - * can be accessed using the usual std::map interface. */ -struct options_group : public std::map { - - bool - has(const std::string& opt_name) const { - return count(opt_name) != 0; - } - - /*! If the options_group contains an option named 'opt_name' - * of type T, returns the value for that option. - * If the option does not exist, throws std::invalid_argument exception. - * Otherwise throws boost::bad_any_cast exception */ - template - const T& - get_as(const std::string& opt_name) const { - - auto it = find(opt_name); - - if(it == end()) { - throw std::invalid_argument("option '" + opt_name + "' missing"); - } - - return it->second.get_as(); - } -}; - -/*! Class to handle a list of several groups */ -using options_list = std::vector; - -/*! Class that stores the values for the defined variables so that they - * can be accessed in a key-value manner. - * This class is derived from std::map, - * therefore all usual std::map operators can be used to access - * its contents */ -struct options_map : public std::map { - - /*! If the options_map contains an option named 'opt_name' - * of type T, returns the value for that option. - * If the option does not exist, throws std::invalid_argument exception. - * Otherwise throws boost::bad_any_cast exception */ - template - T& - get_as(const std::string& opt_name) { - - auto it = find(opt_name); - - if(it == end()) { - throw std::invalid_argument("option '" + opt_name + "' missing"); - } - - return boost::any_cast(it->second); - } -}; - -} // namespace file_options - -#endif /* SCORD_CONFIG_FILE_OPTIONS_DESCRIPTION_HPP */ diff --git a/src/config/file_options/schema.hpp b/src/config/file_options/schema.hpp deleted file mode 100644 index ea825d30..00000000 --- a/src/config/file_options/schema.hpp +++ /dev/null @@ -1,414 +0,0 @@ -/****************************************************************************** - * Copyright 2021, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of scord. - * - * scord is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * scord is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with scord. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -/* - * This file implements the classes necessary to describe the structure - * of a configuration file, as well as the options that are accepted and - * how to process them when found. - * - * A typical configuration file can be defined as follows: - * - * const file_schema config_schema = declare_file({ - * declare_section( - * "section0", // section name - * true, // section is mandatory - * declare_group({ // define a group of related options - * declare_option< - * uint32_t // type for option value to be converted to - * >( - * "option0", // option name in file - * true, // mandatory - * 8, // default value (optional) - * number_parser // converter function - * // std::string -> uint32_t (optional) - * ), - * {{ other option definitions... }} - * }) - * ), - * declare_section( - * "section1", // section name - * true, // section is mandatory - * declare_list({ // define a list of options that can appear - * // several times - * declare_option< - * std::string // type for option value to be converted to - * >( - * "option1", // option name in file - * true // mandatory - * ), - * declare_option< - * std::string // type for option value to be converted to - * >( - * "option2", // option name in file - * false // mandatory - * ) - * }) - * ), - * {{ other section definitions... }} - * }); - * - */ - -#ifndef SCORD_CONFIG_FILE_OPTIONS_SCHEMA_HPP -#define SCORD_CONFIG_FILE_OPTIONS_SCHEMA_HPP - -#include -#include -#include - -//#include "utils.h" - -namespace file_options { - -/*! Special option name that can be used to accept any non-defined - * options in list_schema definitions */ -static const std::string match_any = ".*"; - -// forward declarations -struct section_schema; -struct option_schema; - -enum class opt_type : bool { - optional, - mandatory, -}; - -enum class sec_type : bool { - optional, - mandatory, -}; - -enum class grp_type : bool { - optional, - mandatory, -}; - - -/*! Class to represent a file organization as a key-value map - * of section definitions, where file_schema[section_name] returns - * the parsed section contents. - * - * Since we are implicitly using std::map, all its typical - * functions can be used */ -using file_schema = std::map; - - -/*! Class to handle a group of related options as a key-value map - * where group_schema[option_name] returns the parsed (and converted) - * option_value - * Since we are implicitly using std::map, all its typical - * functions can be used */ -using group_schema = std::map; - - -/*! Class to represent a list of options that can be arbitrarily - * repeated in a section. - * - * Since we inherit from group_schema, which is implicitly using - * std::map, all its typical functions can be used */ -struct list_schema : public group_schema { - // inherit constructors from group_schema - using group_schema::group_schema; -}; - -/*! Abstract base class to handle the semantics of a configuration option. - * By deriving specialized classes from this base class, we can provide - * a simple interface for the user to define an option (see typed_value). */ -struct option_type { - /*! Returns true if a default value was provided for this option when - * constructed */ - virtual bool - has_default_value() const = 0; - - /*! Returns true if a converter function was provided for this option when - * constructed */ - virtual bool - has_converter() const = 0; - - /*! Returns the default value of this function if it was provided - * when constructed. If it was not provided, the return value is - * undefined. */ - virtual boost::any - default_value() const = 0; - - /*! Conversion function from std::string to the type defined for the option - */ - virtual boost::any - convert(const std::string& key, const std::string& value) const = 0; - - /*! Destructor to avoid leaking */ - virtual ~option_type() {} -}; - -/*! Type definition for conversion function */ -template -using ParserFun = std::function; - -/*! Public helper trait for conversion function type */ -template -using converter = ParserFun; - -/*! Concrete class to handle the semantics of a configuration option - * of type T. It stores the default value of type T provided by the user, - * as well as a specialized conversion function std::string -> T to be used, - * when parsing the option's value. */ -template -struct typed_value : public option_type { - - /*! Constructor for an option without default value - * or conversion function */ - typed_value() : m_has_default_value(false), m_has_converter(false) {} - - /*! Constructor for an option with default value - * and no conversion function */ - explicit typed_value(const T default_value) - : m_has_default_value(true), m_default_value(default_value), - m_has_converter(false) {} - - /*! Constructor for an option with a conversion - * function and no default value */ - explicit typed_value(const ParserFun& converter) - : m_has_default_value(false), m_has_converter(true), - m_converter(converter) {} - - /*! Constructor for an option with a default value - * and a conversion function */ - typed_value(const T default_value, const ParserFun& converter) - : m_has_default_value(true), m_default_value(default_value), - m_has_converter(true), m_converter(converter) {} - - /*! Returns true if a default value was provided for this option when - * constructed */ - bool - has_default_value() const override { - return m_has_default_value; - } - - /*! Returns true if a converter function was provided for this option when - * constructed */ - bool - has_converter() const override { - return m_has_converter; - } - - /*! Returns the default value of this function if it was provided - * when constructed. If it was not provided, the return value is - * undefined. */ - boost::any - default_value() const override { - return m_default_value; - } - - /*! If a converter function has been provided for this option, the - * function is applied to 'value' and the result (hopefully of type T) is - * returned to the caller. If no function is provided, an undefined value - * is returned, given that we don't know how to transform std::string to T. - * If the conversion function fails, the exception is propagated to the user - */ - boost::any - convert(const std::string& key, const std::string& value) const override { - - if(m_has_converter) { - return m_converter(key, value); - } - - return {}; - } - - bool m_has_default_value; /*!< True if a default value was provided */ - T m_default_value; /*!< Default value provided by the user */ - bool m_has_converter; /*!< True if a conversion function was provided */ - ParserFun m_converter; /*!< Conversion fucntion provided by the user */ -}; - -/*! Creates a typed_value instance. This function is the primary method - * to create a typed_value instance for a specific type, which can later be - * passed to the group_schema and list_schema constructors. - * - * Overload without default value or conversion function */ -template -std::shared_ptr> -type() { - return std::make_shared>(); -} - -/*! Creates a typed_value instance. This function is the primary method - * to create a typed_value instance for a specific type, which can later be - * passed to the group_schema and list_schema constructors. - * - * Overload with default value only */ -template -std::shared_ptr> -type(const T& default_value) { - return std::make_shared>(default_value); -} - -/*! Creates a typed_value instance. This function is the primary method - * to create a typed_value instance for a specific type, which can later be - * passed to the group_schema and list_schema constructors. - * - * Overload with conversion function only */ -template -std::shared_ptr> -type(const ParserFun& converter) { - return std::make_shared>(converter); -} - -/*! Creates a typed_value instance. This function is the primary method - * to create a typed_value instance for a specific type, which can later be - * passed to the group_schema and list_schema constructors. - * - * Overload with default value and conversion function */ -template -std::shared_ptr> -type(const T& default_value, const ParserFun& converter) { - return std::make_shared>(default_value, converter); -} - -/*! Class used to handle the description of a section */ -struct section_schema { - - using SchemaType = boost::variant; - - section_schema(bool mandatory, SchemaType&& schema) - : m_mandatory(mandatory), m_schema(schema) {} - - bool - is_mandatory() const { - return m_mandatory; - } - - bool m_mandatory; - SchemaType m_schema; -}; - -/*! Class used to handle the description of an option */ -struct option_schema { - - /*! Creates an instance of option_schema and stores whether - * the option is mandatory or not, and also its type */ - option_schema(bool mandatory, std::shared_ptr type) - : m_mandatory(mandatory), m_typed_value(type) {} - - /*! Returns true if the option has been defined as mandatory */ - bool - is_mandatory() const { - return m_mandatory; - } - - /*! Returns true if a default value has been defined for the option */ - bool - has_default_value() const { - return m_typed_value->has_default_value(); - } - - /*! Returns the default value defined for m_typed_value, if any. - * Otherwise, the return value is undefined */ - boost::any - default_value() const { - return m_typed_value->default_value(); - } - - /*! Returns true if a converter function has been defined for m_typed_value - */ - bool - has_converter() const { - return m_typed_value->has_converter(); - } - - /*! Calls the converter function defined for m_typed_value, if any. - * Otherwise, the result is undefined */ - boost::any - convert(const std::string& name, const std::string& value) const { - return m_typed_value->convert(name, value); - } - - bool m_mandatory; /*!< True if the option is mandatory */ - std::shared_ptr - m_typed_value; /*!< Specific semantics for this option */ -}; - -// some helpers to simplify the syntax of a file declaration - -using OptionSchemaType = std::pair; - -template -OptionSchemaType -declare_option(const std::string& name, opt_type ot) { - return {name, {static_cast(ot), type()}}; -} - -template -OptionSchemaType -declare_option(const std::string& name, opt_type ot, const T& default_value) { - return {name, {static_cast(ot), type(default_value)}}; -} - -template -OptionSchemaType -declare_option(const std::string& name, opt_type ot, - const ParserFun& parser) { - return {name, {static_cast(ot), type(parser)}}; -} - -template -OptionSchemaType -declare_option(const std::string& name, opt_type ot, const T& default_value, - const ParserFun& parser) { - return {name, {static_cast(ot), type(default_value, parser)}}; -} - -list_schema -declare_list(const std::initializer_list&& args) { - return {args}; -} - -group_schema -declare_group(const std::initializer_list&& args) { - return {args}; -} - -using SectionSchemaType = std::pair; - -SectionSchemaType -declare_section(const std::string& name, sec_type st, - const group_schema&& schema) { - return {name, {static_cast(st), {schema}}}; -} - -SectionSchemaType -declare_section(const std::string& name, sec_type st, - const list_schema&& schema) { - return {name, {static_cast(st), {schema}}}; -} - -/*! Declares a file_schema as a list of section_schema definitions */ -file_schema -declare_file(const std::initializer_list&& args) { - return {args}; -} - -} // namespace file_options - -#endif /* SCORD_CONFIG_FILE_OPTIONS_SCHEMA_HPP */ diff --git a/src/config/file_options/yaml_parser.hpp b/src/config/file_options/yaml_parser.hpp deleted file mode 100644 index 96a462e9..00000000 --- a/src/config/file_options/yaml_parser.hpp +++ /dev/null @@ -1,244 +0,0 @@ -/****************************************************************************** - * Copyright 2021, Barcelona Supercomputing Center (BSC), Spain - * - * This software was partially supported by the EuroHPC-funded project ADMIRE - * (Project ID: 956748, https://www.admire-eurohpc.eu). - * - * This file is part of scord. - * - * scord is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * scord is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with scord. If not, see . - * - * SPDX-License-Identifier: GPL-3.0-or-later - *****************************************************************************/ - -#ifndef SCORD_CONFIG_FILE_OPTIONS_YAML_PARSER_HPP -#define SCORD_CONFIG_FILE_OPTIONS_YAML_PARSER_HPP - -/* - * This file contains the code to implement a YAML configuration file - * parser, that reads the file structure from a file_schema instance - * and returns an options_map with the parsed values as a result - */ - -#include -#include - -#include "schema.hpp" -#include "options_description.hpp" - -namespace fs = std::filesystem; - -namespace file_options { -namespace detail { - -/*! This class implements a visitor that processes a 'file_schema' instance - * and parses a corresponding YAML-formatted file according to the options - * described in that schema. */ -class yaml_visitor : public boost::static_visitor<> { -public: - /*! Constructor. */ - yaml_visitor(std::string name, const YAML::Node& config, - options_map& opt_map) - : m_name(std::move(name)), m_config(config), m_options_map(opt_map) {} - - /*! Process a 'group_schema'. - * A 'group_schema' is a named section in a 'file_schema' that - * describes a related group of options: - * - * "named-group": [ - * opt0: "foo", - * opt1: "42" - * ] - * - * As such, a 'group_schema' represents a YAML sequence of entries, - * where each entry corresponds to a map of key-value pairs - */ - void - operator()(group_schema& opt_group) const { - - std::set mandatory_options; - std::map default_values; - - // store any mandatory options so that we can check later if - // the user provided them - for(const auto& opt : opt_group) { - - const auto& opt_name = opt.first; - const auto& opt_schema = opt.second; - - if(opt_schema.is_mandatory()) { - mandatory_options.emplace(opt_name); - } - - if(opt_schema.has_default_value()) { - default_values.emplace(opt_name, opt_schema.default_value()); - } - } - - options_group parsed_values; - - for(const auto& entry : m_config) { - for(const auto& kv : entry) { - const auto key = kv.first.as(); - const auto text_value = kv.second.as(); - - auto it = opt_group.find(key); - - if(it == opt_group.end()) { - throw std::invalid_argument("Invalid argument '" + key + - "' in '" + m_name + "'"); - } - - auto& opt_schema = it->second; - - boost::any value; - - if(opt_schema.has_converter()) { - value = opt_schema.convert(key, text_value); - } else { - value = text_value; - } - - parsed_values.emplace(key, option_value(value)); - } - } - - // check whether all mandatory options were provided - for(const auto& opt_name : mandatory_options) { - if(parsed_values.count(opt_name) == 0) { - throw std::invalid_argument("Missing mandatory option '" + - opt_name + "'"); - } - } - - // non-mandatory options not-provided by the user need to be - // initialized to their default values (if any) - for(const auto& v : default_values) { - - const auto& opt_name = v.first; - const auto& opt_value = v.second; - - if(parsed_values.count(opt_name) == 0) { - parsed_values.emplace(opt_name, opt_value); - } - } - - m_options_map.emplace(m_name, std::move(parsed_values)); - } - - /*! Process a 'list_schema'. - * A 'list_schema' is a named section in a 'file_schema' that - * describes a list of repeated group of options, as follows: - * - * "named-list": [ - * [ opt0: "foo", - * opt1: "42", - * opt3: "baz" ], - * - * [ opt0: "bar", - * opt1: "6" ] - * ] - * - * As such, a 'list_schema' represents a YAML sequence of entries, - * where each entry corresponds to another sequence where - * each of its entries is a map of key-value pairs - */ - void - operator()(list_schema& opt_list) const { - (void) opt_list; - - bool accept_all = (opt_list.count(match_any) != 0); - - options_list parsed_values; - - for(const auto& entry : m_config) { - - options_group parsed_group; - - for(const auto& subentry : entry) { - for(const auto& kv : subentry) { - const auto key = kv.first.as(); - const auto text_value = kv.second.as(); - - auto it = opt_list.find(key); - - if(it == opt_list.end()) { - if(!accept_all) { - throw std::invalid_argument("Invalid argument '" + - key + "' in '" + - m_name + "'"); - } - - it = opt_list.find(match_any); - - assert(it != opt_list.end()); - } - - auto& opt_schema = it->second; - - boost::any value; - - if(opt_schema.has_converter()) { - value = opt_schema.convert(key, text_value); - } else { - value = text_value; - } - - parsed_group.emplace(key, option_value(value)); - } - } - - parsed_values.emplace_back(parsed_group); - } - - m_options_map.emplace(m_name, std::move(parsed_values)); - } - - const std::string m_name; - const YAML::Node& m_config; - options_map& m_options_map; -}; - -} // namespace detail - -/*! Parses 'yaml_file' and stores in 'opt_map' the values of any options defined - * in 'schema' */ -void -parse_yaml_file(const fs::path& yaml_file, const file_schema& schema, - options_map& opt_map) { - - const YAML::Node config = YAML::LoadFile(yaml_file.string()); - - for(const auto& opt : schema) { - - auto opt_name = opt.first; - auto opt_schema = opt.second; - - if(!config[opt_name]) { - if(opt_schema.is_mandatory()) { - throw std::invalid_argument("Mandatory section '" + opt_name + - "' is missing"); - } - continue; - } - - boost::apply_visitor( - detail::yaml_visitor(opt_name, config[opt_name], opt_map), - opt_schema.m_schema); - } -} - -} // namespace file_options - -#endif /* SCORD_CONFIG_FILE_OPTIONS_YAML_PARSER_HPP */ diff --git a/src/config/genopts.yml.in b/src/config/genopts.yml.in new file mode 100644 index 00000000..fae2d629 --- /dev/null +++ b/src/config/genopts.yml.in @@ -0,0 +1,14 @@ +genopts: + schema: + copyright_file: "@CMAKE_SOURCE_DIR@/COPYRIGHT_NOTICE" + header_guard: "SCORD_CONFIG_SCHEMA_HPP" + namespace: "scord::config" + output_path: + header: "@CMAKE_CURRENT_BINARY_DIR@/config_options.hpp" + + keywords: + copyright_file: "@CMAKE_SOURCE_DIR@/COPYRIGHT_NOTICE" + header_guard: "SCORD_CONFIG_KEYWORDS_HPP" + namespace: "scord::config::keywords" + output_path: + header: "@CMAKE_CURRENT_BINARY_DIR@/keywords.hpp" -- GitLab From 7addf29f888722e1caa80a07b2c4ed5bdba2763f Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 20 May 2022 20:21:19 +0200 Subject: [PATCH 4/5] Add python3-venv to Docker image --- .gitlab-ci.yml | 3 +-- docker/0.1.0/Dockerfile | 40 +++++++++++++++++++++------------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fd641fed..b5690ac4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ # Compilation of scord and execution of tests -image: gekkofs/scord:0.1.0 +image: bscstorage/scord:0.1.0 stages: - build @@ -39,4 +39,3 @@ test: - pkill -9 scord cache: key: $CI_COMMIT_REF_SLUG - diff --git a/docker/0.1.0/Dockerfile b/docker/0.1.0/Dockerfile index e7eb72b0..65a59ad9 100644 --- a/docker/0.1.0/Dockerfile +++ b/docker/0.1.0/Dockerfile @@ -7,28 +7,30 @@ ENV INSTALL_PATH /usr/local RUN apt-get update && \ apt-get install -y --no-install-recommends \ - git \ - curl \ - ca-certificates \ - libtool \ - pkg-config \ - make \ - automake \ - gcc \ - g++ \ - procps \ - # AGIOS dependencies - libconfig-dev \ - # Mercury dependencies - libltdl-dev \ - lbzip2 \ + git \ + curl \ + ca-certificates \ + libtool \ + pkg-config \ + make \ + automake \ + gcc \ + g++ \ + procps \ + # AGIOS dependencies + libconfig-dev \ + # Mercury dependencies + libltdl-dev \ + lbzip2 \ # Margo dependencies \ libjson-c-dev \ - # GekkoFS dependencies - libboost-program-options-dev \ - uuid-dev \ + # GekkoFS dependencies + libboost-program-options-dev \ + uuid-dev \ python3 \ - libyaml-dev libcurl4-openssl-dev procps && \ + libyaml-dev libcurl4-openssl-dev procps \ + # genopts dependencies + python3-venv && \ # install cmake 3.14 since it's needed for some dependencies curl -OL https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-Linux-x86_64.sh && \ chmod u+x ./cmake-3.23.1-Linux-x86_64.sh && \ -- GitLab From f82930582c9db5d33c03d3bee899ec8d27dc847d Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 20 May 2022 20:25:59 +0200 Subject: [PATCH 5/5] Add missing files --- COPYRIGHT_NOTICE | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 COPYRIGHT_NOTICE diff --git a/COPYRIGHT_NOTICE b/COPYRIGHT_NOTICE new file mode 100644 index 00000000..a3f50b7b --- /dev/null +++ b/COPYRIGHT_NOTICE @@ -0,0 +1,23 @@ +/****************************************************************************** + * Copyright 2021-2022, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of scord. + * + * scord is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * scord is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with scord. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ -- GitLab