From ac9bf62be1f2c1f1ae561a246c1e74350605b330 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 3 Feb 2023 16:07:33 +0100 Subject: [PATCH 1/2] Spack: Add repository and package definitions --- spack/packages/cargo/package.py | 91 +++++++++++++++++++++++++++++++++ spack/repo.yaml | 2 + 2 files changed, 93 insertions(+) create mode 100644 spack/packages/cargo/package.py create mode 100644 spack/repo.yaml diff --git a/spack/packages/cargo/package.py b/spack/packages/cargo/package.py new file mode 100644 index 0000000..6b199e5 --- /dev/null +++ b/spack/packages/cargo/package.py @@ -0,0 +1,91 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# ---------------------------------------------------------------------------- +# If you submit this package back to Spack as a pull request, +# please first remove this boilerplate and all FIXME comments. +# +# This is a template package file for Spack. We've put "FIXME" +# next to all the things you'll want to change. Once you've handled +# them, you can save this file and test your package like this: +# +# spack install cargo +# +# You can edit this file again by typing: +# +# spack edit cargo +# +# See the Spack documentation for more information on packaging. +# ---------------------------------------------------------------------------- + +from spack.package import * + + +class Cargo(CMakePackage): + """A parallel data stager for malleable applications.""" + + homepage = "https://storage.bsc.es/gitlab/hpc/cargo" + url = "https://storage.bsc.es/gitlab/hpc/cargo/-/archive/v0.1.0/cargo-v0.1.0.tar.bz2" + git = "https://storage.bsc.es/gitlab/hpc/cargo.git" + + maintainers("alberto-miranda") + + # available versions + version("latest", branch="main") + version("0.1.0", sha256="981d00adefbc2ea530f57f8428bd7980e4aab2993a86d8ae4274334c8f055bdb") + + # build variants + variant('build_type', + default='Release', + description='CMake build type', + values=('Debug', 'Release', 'RelWithDebInfo', 'ASan')) + + variant('tests', + default=False, + description='Build and run Cargo tests') + + variant("ofi", + default=True, + when="@0.1.0:", + description="Use OFI libfabric as transport library") + + variant("ucx", + default=False, + when="@0.1.0:", + description="Use UCX as transport library") + + + # general dependencies + depends_on("cmake@3.19", type='build') + + # specific dependencies + # v0.1.0+ + depends_on("mpi", when='@0.1.0:') + depends_on("argobots@1.1", when='@0.1.0:') + depends_on("mochi-margo@0.9.8", when='@0.1.0:') + depends_on("mochi-thallium@0.10.1", when='@0.1.0:') + depends_on("boost@1.71 +program_options +mpi", when='@0.1.0:') + depends_on("boost@1.71 +iostreams", when='@0.1.0: +tests') + + with when("@0.1.0: +ofi"): + depends_on("libfabric@1.14.0 fabrics=sockets,tcp,rxm") + depends_on("mercury@2.1.0 +ofi") + + with when("@0.1.0: +ucx"): + depends_on("ucx@1.12.0") + depends_on("mercury@2.1.0 +ucx") + + def cmake_args(self): + """Setup Cargo CMake arguments""" + cmake_args = [ + self.define_from_variant('CARGO_BUILD_TESTS', 'tests') + ] + return cmake_args + + def check(self): + """Run tests""" + with working_dir(self.build_directory): + make("test", parallel=False) + diff --git a/spack/repo.yaml b/spack/repo.yaml new file mode 100644 index 0000000..f4fc33f --- /dev/null +++ b/spack/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: 'admire' -- GitLab From 85b3178bc265c242c1201054964a0c50c3efa4d6 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 3 Feb 2023 16:31:24 +0100 Subject: [PATCH 2/2] docs: Update README.md --- README.md | 113 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 102 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index e315a0b..f93ac25 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# cargo +# Cargo Cargo is a HPC data staging service that runs alongside applications helping them to transfer data in parallel between local and shared storage tiers. @@ -8,16 +8,106 @@ them to transfer data in parallel between local and shared storage tiers. > This software was partially supported by the EuroHPC-funded project ADMIRE > (Project ID: 956748, https://www.admire-eurohpc.eu). -## Dependencies -- Margo (tested with v0.9.8) -- Argobots (tested with v1.1rc2) -- Mercury (tested with v2.1.0rc3) -- Thallium (tested with 5daa9a909d1309620ed874832cf8075644727a8a) -- libboost (tested with 1.71.0) -- MPI (tested with OpenMPI 4.0.3) +## Building Cargo -## Installation +### Building Cargo and its dependencies with Spack + +Cargo and its dependencies can be built using +[Spack](https://github.com/spack/spack). If you already have Spack, make sure +you have the latest release. If you use a clone of the Spack `develop` +branch, be sure to pull the latest changes. + +#### Install Spack + +If you haven't already, install Spack with the following commands: + +```shell +$ git clone -c feature.manyFiles=true https://github.com/spack/spack +``` + +This will create a directory called `spack` in your machine. Once you have +cloned Spack, we recommend sourcing the appropriate script for your shell. +This will add Spack to your PATH and enable the use of the `spack` command: + +```shell +# For bash/zsh/sh +$ . spack/share/spack/setup-env.sh + +# For tcsh/csh +$ source spack/share/spack/setup-env.csh + +# For fish +$ . spack/share/spack/setup-env.fish +``` + +Since `Cargo` is not yet available in the official Spack repositories, you need +to add the Cargo Spack repository to the Spack namespace in your machine. To do +that, download the `spack/` directory in the `Cargo` repository's root to your +machine (e.g. under `~/projects/cargo/spack`) and execute the following: + +```shell +spack repo add ~/projects/cargo/spack/ +``` + +You should now be able to fetch information from the `Cargo` package using +Spack: + +```shell +spack info cargo +``` + +You are now ready to install `Cargo`: + +```shell +spack install cargo +``` + +Include or remove variants with Spack when a custom `Cargo` build is desired. +The available variants are listed below: + + +| Variant | Command | Default | Description | +|---------|-------------|---------|------------------------------------| +| OFI | `cargo+ofi` | True | Use libfabric as transport library | +| UCX | `cargo+ucx` | False | Use ucx as transport library | + + +> **Attention** +> +> The initial install could take a while as Spack will install build +> dependencies (autoconf, automake, m4, libtool, and pkg-config) as well as +> any dependencies of dependencies (cmake, perl, etc.) if you don’t already +> have these dependencies installed through Spack. + +After the installation completes, remember that you first need to load +`Cargo` in order to use it: + +```shell +spack load cargo +``` + +### Building Cargo manually + +If you prefer to build and install `Cargo` from sources, you can also do so. +For the build process to work correctly, the dependencies below will +need to be available in your system: + +| Dependency | Version | +|----------------------------------------------------|---------------------------| +| Margo | v0.9.8+ | +| Argobots | v1.1+ | +| Mercury | v2.1.0+ | +| Thallium | v0.10.1+ | +| libfabric (if `CARGO_TRANSPORT_LIBRARY=libfabric`) | v0.10.1+ | +| ucx (if `CARGO_TRANSPORT_LIBRARY=ucx`) | v0.10.1+ | +| boost program_options | v1.71.0+ | +| boost mpi | v1.71.0+ | +| boost iostreams (optional, for testing) | v1.71.0+ | +| MPI | tested with OpenMPI 4.0.3 | + +Once all dependencies are available, you can download build and install +`Cargo` with the following commands: ```shell ## clone the repository @@ -34,13 +124,14 @@ mkdir build && cd build cmake \ -DCMAKE_PREFIX_PATH:STRING="${PREFIX};${CMAKE_PREFIX_PATH}" \ -DCMAKE_INSTALL_PREFIX:STRING="${INSTALL_DIR}" \ + -DCARGO_TRANSPORT_LIBRARY:STRING=libfabric \ -DCARGO_BUILD_TESTS:BOOL=ON \ .. make -j8 install ``` -These commands will generate and install the Cargo server binary -(`${INSTALL_DIR}/bin/cargo`) as well as the Cargo interface +These commands will generate and install the `Cargo` server binary +(`${INSTALL_DIR}/bin/cargo`) as well as the `Cargo` interface library (`${INSTALL_DIR}/lib/libcargo.so`) and its headers (`${INSTALL_DIR}/include/cargo/*`). -- GitLab