From 6539d3816f73cce59cf21a7db9d25c43067f27d9 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 12 Jun 2025 15:09:43 +0200 Subject: [PATCH 1/3] updated gekkofs deps --- spack/packages/cargo/package.py | 34 +++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/spack/packages/cargo/package.py b/spack/packages/cargo/package.py index 7209651..add9d51 100644 --- a/spack/packages/cargo/package.py +++ b/spack/packages/cargo/package.py @@ -54,40 +54,46 @@ class Cargo(CMakePackage): variant("ofi", default=True, - when="@0.1.0:", + when="@0.1.0:,latest", description="Use OFI libfabric as transport library") variant("ucx", default=False, - when="@0.1.0:", + when="@0.1.0:,latest", description="Use UCX as transport library") + variant("gekkofs", + default=False, + when="@0.1.0:,latest", + description="Use gekkofs user library") # general dependencies depends_on("c", type="build") depends_on("cxx", type="build") - depends_on("cmake@3.19", type='build') + depends_on("cmake@3.19:", type='build') depends_on("mpi") #specific dependencies # v0.2.0+ - depends_on("boost@1.71: +mpi", when='@0.2.0:') + depends_on("boost@1.71: +mpi", when='@0.2.0:,latest') # specific dependencies # v0.1.0+ - depends_on("argobots@1.1", when='@0.1.0:') - depends_on("mochi-margo@0.14.1:", when='@0.1.0:') - depends_on("mochi-thallium@0.11.3:", when='@0.1.0:') - depends_on("boost@1.71 +program_options +mpi", when='@:0.1.0') - depends_on("boost@1.71 +iostreams", when='@0.1.0: +tests') + depends_on("argobots@1.1:", when='@0.1.0:,latest') + depends_on("mochi-margo@0.18.3", when='@0.1.0:,latest') + depends_on("mochi-thallium@0.11.3:", when='@0.1.0:,latest') + depends_on("boost@1.71 +program_options +mpi", when='@:0.1.0,latest') + depends_on("boost@1.71 +iostreams", when='@0.1.0:,latest +tests') - with when("@0.1.0: +ofi"): - depends_on("libfabric@1.14.0 fabrics=sockets,tcp,rxm") - depends_on("mercury@2.1.0 +ofi") + with when("@0.1.0:,latest +ofi"): + depends_on("libfabric@1.20.1 fabrics=sockets,tcp,rxm") + depends_on("mercury@2.4.0 +ofi") - with when("@0.1.0: +ucx"): + with when("@0.1.0:,latest +ucx"): depends_on("ucx@1.12.0") - depends_on("mercury@2.1.0 +ucx") + depends_on("mercury@2.4.0 +ucx") + + depends_on("gekkofs@latest", when='@0.1.0:,latest +gekkofs', type='link') def cmake_args(self): """Setup Cargo CMake arguments""" -- GitLab From 8983503b599da63ac2de6ee6e27a87c0311a240b Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 13 Jun 2025 07:08:50 +0200 Subject: [PATCH 2/3] mod mochi-thallium due to missing c --- spack/packages/cargo/package.py | 8 +-- spack/packages/mochi-thallium/package.py | 89 ++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 spack/packages/mochi-thallium/package.py diff --git a/spack/packages/cargo/package.py b/spack/packages/cargo/package.py index add9d51..5eeaf04 100644 --- a/spack/packages/cargo/package.py +++ b/spack/packages/cargo/package.py @@ -70,7 +70,7 @@ class Cargo(CMakePackage): # general dependencies depends_on("c", type="build") depends_on("cxx", type="build") - depends_on("cmake@3.19:", type='build') + depends_on('cmake@3.15: -doc -ncurses -qtgui', type='build') depends_on("mpi") #specific dependencies @@ -78,10 +78,8 @@ class Cargo(CMakePackage): depends_on("boost@1.71: +mpi", when='@0.2.0:,latest') # specific dependencies - # v0.1.0+ - depends_on("argobots@1.1:", when='@0.1.0:,latest') - depends_on("mochi-margo@0.18.3", when='@0.1.0:,latest') - depends_on("mochi-thallium@0.11.3:", when='@0.1.0:,latest') + # v0.1.0+ + depends_on("mochi-thallium@0.15.0:", when='@0.1.0:,latest') depends_on("boost@1.71 +program_options +mpi", when='@:0.1.0,latest') depends_on("boost@1.71 +iostreams", when='@0.1.0:,latest +tests') diff --git a/spack/packages/mochi-thallium/package.py b/spack/packages/mochi-thallium/package.py new file mode 100644 index 0000000..50e5d3e --- /dev/null +++ b/spack/packages/mochi-thallium/package.py @@ -0,0 +1,89 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack_repo.builtin.build_systems.cmake import CMakePackage + +from spack.package import * + + +class MochiThallium(CMakePackage): + """A Mochi C++14 library wrapping Margo, Mercury, + and Argobots and providing an object-oriented way to use these libraries.""" + + homepage = "https://github.com/mochi-hpc/mochi-thallium" + url = "https://github.com/mochi-hpc/mochi-thallium/archive/refs/tags/v0.8.5.tar.gz" + git = "https://github.com/mochi-hpc/mochi-thallium.git" + + maintainers("mdorier") + + version("main", branch="main") + version("0.15.0", sha256="a7872e926e97fdf80a67c8e44f1217a959c689763dbcf9712abd913d1ef23bdf") + version("0.14.6", sha256="dfdd39fc840a82a69c1698e764239e8aa0a5573f677a52fb2bdd6bffd529a232") + version("0.14.5", sha256="8a13f1adb6a549053f56b46235ea81ed9c047cd702b8980035fc81be3ea942e3") + version("0.14.4", sha256="bca33ef4af640581a1729606b708974a955a2a2a2f3817ee110c2c9da2da9a99") + version("0.14.3", sha256="b37b8fa9976471950e9d74e0269c2dc80ca5353e97c7ee4603460077fab28ca3") + version("0.14.2", sha256="7dc03a84845aa4b902c0b52d8384dd1b9bef02b53f880efb02ec58a12d8c6381") + version("0.14.1", sha256="0de7b7b5b517af552ababab7b5ef973207515398f7fd9685b3f6841432913c7b") + version("0.14.0", sha256="3af3c2e4cae15a256e76df89ed9ad46ced68ca9b045216a9510f563e96722104") + version("0.13.1", sha256="8166c412ebeb58898198069adbaf126362cffb2ba80ccf3c24b5cead0368acfa") + version("0.13.0", sha256="29f50b338c247ce5945ea90241ad938b951c4bac8af070cc3136f10f309ae542") + version("0.12.0", sha256="cbb6ea8f479d74a4310847ffd7eb4fb11107732540ebc13b5989b7c9809f6d06") + version("0.11.3", sha256="d1ffd7ee1ccbcfb00f246cb29c5bc2560e59f8808609cbc19b7098aa8fc903c4") + version("0.11.2", sha256="4f1e57ca843b7592525c179dec73bfb603a27fbda4feaf028d636e05c1b38e36") + version("0.11.1", sha256="be99bec2309ce1945a777fba720175f409972cbf27b73388728a740d6406a040") + version("0.11.0", sha256="c216310fdef9281e1c7e3264c148c560d7f5edd15816d35866efcc543185b7ee") + version("0.10.1", sha256="5a8dc1f1622f4186b02fbabd47a8a33ca6be3d07757010f3d63d30e9f74fec8c") + version("0.10.0", sha256="5319e25a42deab7c639e980885fe3be717cda2c2c693a1906f5a6c79b31edef8") + version("0.9.1", sha256="dee884d0e054c838807f9c17781acfa99b26e3be1cc527bf09ceaa997336b3e4") + version("0.9", sha256="6b867b73f5dd76ea160d83782980149f33ae3567c370cee63d945e2e94609331") + version("0.8.5", sha256="2d6d1fd97ad5b38c848ece6428c27400f752d57254324bfd0f1ea660d6a815a6") + version("0.8.4", sha256="cb3f98a399664f41917a9a046fac3058f6ec53c85174dd97d5befd361bec0727") + version("0.8.3", sha256="47b7837796ebd362b993cda5131912372c4abbc17a6967d177e0c3687c44ab7d") + version("0.8.2", sha256="c90e3551189bae3552aab9089ce6343995736d734199c762bb8fa790968092ed") + version("0.8.1", sha256="3c7a72de3f77917432f9e751559f413845a68563477cc29bd127ca7126c5f64a") + version("0.8", sha256="3032196b1e5d7031fa4f79e81a44503fd02bca3abdc25f6ae0a5f3943e6397e4") + version("0.7", sha256="173be344445b36a6e21abc192ef228fb7ca9dbc33d4484f284b5f4d31c79b7de") + version("0.6.1", sha256="3e13b7d615fca827baa7cbee518f33ee908752c394361ede7668f4c47770f9f3") + version("0.6", sha256="849db7c6297f8dcbb7786a6c3a11db05b316b5f7f028e52fbab439a84986562f") + version("0.5.4", sha256="e5dbd63c8ed9e5bb7548d9a5d04c240482115a245e1312052ca3c17b2fa4e21b") + version("0.5.3", sha256="372567f4203840d98543da9f3d5ef3c7989735b46e60337205578d5c93cf4e02") + version("0.5.2", sha256="fe4f92880a2e574456afd934cede109d973055db9ca2a4a915501d635e42b3e7") + version("0.5.1", sha256="e43abe03c91e38eb865fe6179f389a66cc38f2962f6a5a7ac93b01259f15aa18") + version("0.5", sha256="0f85bad3680e94af734eaf063ec08620d4c16e1c8f0c4f85ae1ba010fbac958c") + version("0.4.2", sha256="cf04efbf01bcbecac121fbc5f5c0d40dca2074638c0682c86f55e72121f64829") + version("0.4.1", sha256="65136c01f276be70592e600a8942cee76164ea4c9646f2deecc9fd83b3c5bcfc") + version("0.4", sha256="51b7efe8706f47a1508624bbfba12d800a8374ac1db076897c7d01d85621d30f") + version("0.3.4", sha256="1f0e56c4a8d1285dcc288a50dadd7803a9cd0a311dd7798fec0b0800877d933a") + version("0.3.3", sha256="33a31d5ccba781343f0ec18396f895e5ed60d9a52b35d3667698754e2a8a639a") + version("0.3.2", sha256="7d90269101c881af40b73b0ffe1d55baa7203075b277fe8763735cc52278cee5") + version("0.3.1", sha256="61403b1ba5f4d205408e6a7e04c785df6dea02f59fe9fa1742db05aa752cc8a0") + version("0.3", sha256="4f9f78e52c1725f6ea5f933d7548bde36729dd9eff08f58fe7fe40682bc5f748") + + variant( + "cereal", + default=True, + description="Use the cereal library for serialization", + when="@0.4.1:", + ) + conflicts("~cereal", when="@0.14.0:", msg="Thallium 0.14.0 and above requires Cereal") + depends_on("c", type="build") + depends_on("cxx", type="build") # generated + + depends_on("pkgconfig", type=("build")) + depends_on("mochi-margo@0.18.0:", when="@0.14.0:") + depends_on("mochi-margo@0.12.0:", when="@0.11.2:") + depends_on("mochi-margo@0.9.8:", when="@0.10.0:") + depends_on("mochi-margo@0.7:", when="@0.7:") + depends_on("mochi-margo@0.6:", when="@0.5:") + depends_on("mochi-margo@0.5:", when="@0.4:0.4.2") + depends_on("mochi-margo@0.4:", when="@:0.3.4") + with when("+cereal"): + depends_on("cereal@:1.3.0", when="@0.4.1:0.10.0") + depends_on("cereal@1.3.1:", when="@0.10.1:") + # thallium relies on std::decay_t + conflicts("%gcc@:4.9.0") + + def cmake_args(self): + args = [] + args.append(self.define_from_variant("ENABLE_CEREAL", "cereal")) + return args -- GitLab From f1902618febac3048a3bd14f9e78838ce69a02d7 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 13 Jun 2025 14:08:15 +0200 Subject: [PATCH 3/3] updated cargo with fmt, removed segfault --- CMakeLists.txt | 149 ++++++++++++++++++++++++++++++++------- lib/fmt_formatters.hpp | 1 + src/parallel_request.hpp | 18 +++-- 3 files changed, 137 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ea946c..786f534 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,108 @@ project( set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) +#[=======================================================================[.rst: + + include_from_source(contentName ...) + +The ``include_from_source()`` function ensures that ``contentName`` is +populated and potentially added to the build by the time it returns. + +**Options:** + + ``SOURCE_DIR ``: Source directory into which downloaded contents reside. + This must point to an existing directory where the external project has + already been unpacked or cloned/checked out. If ```` doesn't exist, + the source code will be retrieved. + + ``GIT_REPOSITORY `` + URL of the git repository. Any URL understood by the ``git`` command + may be used. + + ``GIT_TAG `` + Git branch name, tag or commit hash. Note that branch names and tags should + generally be specified as remote names (i.e. origin/myBranch rather than + simply myBranch). This ensures that if the remote end has its tag moved or + branch rebased or history rewritten, the local clone will still be updated + correctly. In general, however, specifying a commit hash should be + preferred for a number of reasons: + + If the local clone already has the commit corresponding to the hash, no git + fetch needs to be performed to check for changes each time CMake is re-run. + This can result in a significant speed up if many external projects are + being used. + + Using a specific git hash ensures that the main project's own history is + fully traceable to a specific point in the external project's evolution. + If a branch or tag name is used instead, then checking out a specific + commit of the main project doesn't necessarily pin the whole build to a + specific point in the life of the external project. The lack of such + deterministic behavior makes the main project lose traceability and + repeatability. + + NOTE: If both ``SOURCE_DIR`` and ``GIT_REPOSITORY`` are specified, + ``SOURCE_DIR`` will be the preferred location to populate ``contentName`` + from. If ``SOURCE_DIR`` doesn't exist, the function will fall back to the + location defined by ``GIT_REPOSITORY``. + +#]=======================================================================] +function(include_from_source contentName) + + set(OPTIONS) + set(SINGLE_VALUE MESSAGE SOURCE_DIR GIT_REPOSITORY GIT_TAG) + set(MULTI_VALUE) + + cmake_parse_arguments(ARGS "${OPTIONS}" "${SINGLE_VALUE}" "${MULTI_VALUE}" ${ARGN}) + + if (ARGS_MESSAGE) + message(STATUS ${ARGS_MESSAGE}) + endif () + + include(FetchContent) + + if (EXISTS ${ARGS_SOURCE_DIR}) + file(GLOB_RECURSE SOURCE_FILES "${ARGS_SOURCE_DIR}/*") + if (SOURCE_FILES STREQUAL "") + message(FATAL_ERROR + "The '${ARGS_SOURCE_DIR}' source directory appears " + "to be empty. If it corresponds to a git submodule it may not have " + "been properly initialized. Running:\n" + " 'git submodule update --init --recursive'\n" + "may fix the issue. If the directory corresponds to a manually " + "downloaded dependency, please download it again.") + endif () + + message(STATUS "Found source directory for '${contentName}'. Building.") + FetchContent_Declare( + ${contentName} + SOURCE_DIR ${ARGS_SOURCE_DIR} + ) + else () + message(STATUS + "Source directory for '${contentName}' not found.\n" + "Downloading and building from remote Git repository.") + + if (NOT ARGS_GIT_REPOSITORY) + message(FATAL_ERROR "GIT_REPOSITORY for \"${contentName}\" not defined") + endif () + + if (NOT ARGS_GIT_TAG) + message(FATAL_ERROR "GIT_TAG for \"${contentName}\" not defined") + endif () + + FetchContent_Declare( + ${contentName} + GIT_REPOSITORY ${ARGS_GIT_REPOSITORY} + GIT_TAG ${ARGS_GIT_TAG} + GIT_SHALLOW ON + GIT_PROGRESS ON + ) + endif () + + FetchContent_MakeAvailable(${contentName}) +endfunction() + + # Set default build type and also populate a list of available options get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) @@ -209,40 +311,35 @@ message(STATUS "[${PROJECT_NAME}] Checking for Thallium") find_package(Thallium REQUIRED) ### {fmt}: required for sensible output formatting -message(STATUS "[${PROJECT_NAME}] Downloading and building {fmt}") -FetchContent_Declare( - fmt - GIT_REPOSITORY https://github.com/fmtlib/fmt - GIT_TAG 10.2.1 - GIT_SHALLOW ON - GIT_PROGRESS ON +set(FMT_INSTALL OFF) +include_from_source(fmt + MESSAGE "[${PROJECT_NAME}] Searching for {fmt}" + SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/fmt + GIT_REPOSITORY https://github.com/fmtlib/fmt + GIT_TAG 40626af88bd7df9a5fb80be7b25ac85b122d6c21 # v11.2.0 ) -FetchContent_MakeAvailable(fmt) -set_target_properties(fmt PROPERTIES POSITION_INDEPENDENT_CODE ON) +# ensure that fmt is linked as PIC +set_property(TARGET fmt PROPERTY POSITION_INDEPENDENT_CODE ON) ### spdlog: required for logging -message(STATUS "[${PROJECT_NAME}] Downloading and building spdlog") -FetchContent_Declare( - spdlog - GIT_REPOSITORY https://github.com/gabime/spdlog - GIT_TAG v1.14.0 - GIT_SHALLOW ON - GIT_PROGRESS ON +include_from_source(spdlog + MESSAGE "[${PROJECT_NAME}] Searching for spdlog" + SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/spdlog + GIT_REPOSITORY https://github.com/gabime/spdlog.git + GIT_TAG 6fa36017cfd5731d617e1a934f0e5ea9c4445b13 # v1.15.3 ) -FetchContent_MakeAvailable(spdlog) -set_target_properties(spdlog PROPERTIES POSITION_INDEPENDENT_CODE ON) +# ensure that spdlog is linked as PIC +set_property(TARGET spdlog PROPERTY POSITION_INDEPENDENT_CODE ON) ### CLI11: used for parsing command-line options -message(STATUS "[${PROJECT_NAME}] Searching for CLI11") -FetchContent_Declare(cli11 -GIT_REPOSITORY https://github.com/CLIUtils/CLI11 -GIT_TAG 291c58789c031208f08f4f261a858b5b7083e8e2 # v2.3.2 -GIT_SHALLOW ON -GIT_PROGRESS ON +include_from_source(cli11 + MESSAGE "[${PROJECT_NAME}] Searching for CLI11" + SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/CLI11 + GIT_REPOSITORY https://github.com/CLIUtils/CLI11 + GIT_TAG v2.5.0 ) -FetchContent_MakeAvailable(cli11) ### expected: required for using tl::expected in the C++ library implementation ### until std::expected makes it to C++ @@ -309,7 +406,7 @@ FetchContent_Declare( ) FetchContent_GetProperties(asio) if(NOT asio_POPULATED) - FetchContent_Populate(asio) + FetchContent_MakeAvailable(asio) endif() add_library(asio INTERFACE) diff --git a/lib/fmt_formatters.hpp b/lib/fmt_formatters.hpp index 7ab61be..ee9a6af 100644 --- a/lib/fmt_formatters.hpp +++ b/lib/fmt_formatters.hpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include "cargo/error.hpp" diff --git a/src/parallel_request.hpp b/src/parallel_request.hpp index 4413894..fca7592 100644 --- a/src/parallel_request.hpp +++ b/src/parallel_request.hpp @@ -78,7 +78,8 @@ public: bw() const; void - update(std::string name, transfer_state s, float bw, std::optional ec) noexcept; + update(std::string name, transfer_state s, float bw, + std::optional ec) noexcept; private: std::string m_name; @@ -107,8 +108,8 @@ public: bw() const; void - bw (float bw); - + bw(float bw); + private: std::string m_name; transfer_state m_state{transfer_state::pending}; @@ -140,8 +141,15 @@ struct fmt::formatter : formatter { } }; - const auto str = fmt::format("{{state: {}, bw: {}, error_code: {}}}", - state_name(s), s.bw(), *s.error()); + std::string str = ""; + if(s.error()) { + str = + fmt::format("{{state: {}, bw: {}, error_code: {}}}", + state_name(s), s.bw(), *s.error()); + } else { + str = + fmt::format("{{state: {}, bw: {}}}", state_name(s), s.bw()); + } return formatter::format(str, ctx); } }; -- GitLab