From c1cf7c0c5b4f09402dea61fcb4285a28ec8a96a3 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 13 Feb 2023 07:25:55 +0100 Subject: [PATCH 1/2] Add Spack support --- spack/packages/agios/package.py | 39 ++++++++++ spack/packages/hiredis/package.py | 40 ++++++++++ spack/packages/redis-plus-plus/package.py | 38 ++++++++++ spack/packages/scord/package.py | 90 +++++++++++++++++++++++ spack/repo.yaml | 2 + 5 files changed, 209 insertions(+) create mode 100644 spack/packages/agios/package.py create mode 100644 spack/packages/hiredis/package.py create mode 100644 spack/packages/redis-plus-plus/package.py create mode 100644 spack/packages/scord/package.py create mode 100644 spack/repo.yaml diff --git a/spack/packages/agios/package.py b/spack/packages/agios/package.py new file mode 100644 index 00000000..e7faa911 --- /dev/null +++ b/spack/packages/agios/package.py @@ -0,0 +1,39 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# ---------------------------------------------------------------------------- +# If you submit this package back to Spack as a pull request, +# please first remove this boilerplate and all FIXME comments. +# +# This is a template package file for Spack. We've put "FIXME" +# next to all the things you'll want to change. Once you've handled +# them, you can save this file and test your package like this: +# +# spack install agios +# +# You can edit this file again by typing: +# +# spack edit agios +# +# See the Spack documentation for more information on packaging. +# ---------------------------------------------------------------------------- + +from spack.package import * + + +class Agios(CMakePackage): + """AGIOS: an I/O request scheduling library at file level.""" + + + homepage = 'https://github.com/francielizanon/agios' + url = 'https://github.com/jeanbez/agios/archive/refs/tags/v1.0.tar.gz' + git = 'https://github.com/francielizanon/agios.git' + + version('latest', branch='development') + version('1.0', sha256='e8383a6ab0180ae8ba9bb2deb1c65d90c00583c3d6e77c70c415de8a98534efd') + + # general dependencies + depends_on('cmake@3.5', type='build') + depends_on('libconfig') diff --git a/spack/packages/hiredis/package.py b/spack/packages/hiredis/package.py new file mode 100644 index 00000000..6b9c804f --- /dev/null +++ b/spack/packages/hiredis/package.py @@ -0,0 +1,40 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# ---------------------------------------------------------------------------- +# If you submit this package back to Spack as a pull request, +# please first remove this boilerplate and all FIXME comments. +# +# This is a template package file for Spack. We've put "FIXME" +# next to all the things you'll want to change. Once you've handled +# them, you can save this file and test your package like this: +# +# spack install hiredis +# +# You can edit this file again by typing: +# +# spack edit hiredis +# +# See the Spack documentation for more information on packaging. +# ---------------------------------------------------------------------------- + +from spack.package import * + + +class Hiredis(MakefilePackage): + """A minimalistic C client library for the Redis database.""" + + homepage = "https://github.com/redis/hiredis" + url = "https://github.com/redis/hiredis/archive/refs/tags/v1.1.0.tar.gz" + git = "https://github.com/redis/hiredis" + + version("1.1.0", sha256='fe6d21741ec7f3fc9df409d921f47dfc73a4d8ff64f4ac6f1d95f951bf7f53d6') + + # FIXME: Add dependencies if required. + depends_on("openssl") + + def edit(self, spec, prefix): + env['PREFIX'] = prefix + env['USE_SSL'] = '1' diff --git a/spack/packages/redis-plus-plus/package.py b/spack/packages/redis-plus-plus/package.py new file mode 100644 index 00000000..cb357a70 --- /dev/null +++ b/spack/packages/redis-plus-plus/package.py @@ -0,0 +1,38 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# ---------------------------------------------------------------------------- +# If you submit this package back to Spack as a pull request, +# please first remove this boilerplate and all FIXME comments. +# +# This is a template package file for Spack. We've put "FIXME" +# next to all the things you'll want to change. Once you've handled +# them, you can save this file and test your package like this: +# +# spack install redis-plus-plus +# +# You can edit this file again by typing: +# +# spack edit redis-plus-plus +# +# See the Spack documentation for more information on packaging. +# ---------------------------------------------------------------------------- + +from spack.package import * + + +class RedisPlusPlus(CMakePackage): + """A C++ client library for Redis""" + + # FIXME: Add a proper url for your package's homepage here. + homepage = 'https://github.com/sewenew/redis-plus-plus' + url = 'https://github.com/sewenew/redis-plus-plus/archive/refs/tags/1.3.3.zip' + git = 'https://github.com/sewenew/redis-plus-plus' + + version('1.3.3', sha256='0bb1b5e459817d8edc53ecf6e893d82c0ea4b459') + + depends_on('cmake@3.2:', type='build') + + depends_on('hiredis@1.1.0:', when='@1.3.3:') diff --git a/spack/packages/scord/package.py b/spack/packages/scord/package.py new file mode 100644 index 00000000..23f2375a --- /dev/null +++ b/spack/packages/scord/package.py @@ -0,0 +1,90 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# ---------------------------------------------------------------------------- +# If you submit this package back to Spack as a pull request, +# please first remove this boilerplate and all FIXME comments. +# +# This is a template package file for Spack. We've put "FIXME" +# next to all the things you'll want to change. Once you've handled +# them, you can save this file and test your package like this: +# +# spack install scord +# +# You can edit this file again by typing: +# +# spack edit scord +# +# See the Spack documentation for more information on packaging. +# ---------------------------------------------------------------------------- + +from spack.package import * + + +class Scord(CMakePackage): + """A parallel data stager for malleable applications.""" + + homepage = "https://storage.bsc.es/gitlab/eu/admire/io-scheduler" + url = "https://storage.bsc.es/gitlab/eu/admire/io-scheduler/-/archive/v0.2.0/scord-v0.2.0.tar.bz2" + git = "https://storage.bsc.es/gitlab/eu/admire/io-scheduler.git" + + maintainers("alberto-miranda") + + # available versions + version("latest", branch="main") + version("0.2.0", sha256="cfec167dbba00a62977bde63e6ed23512bd5d053") + + # build variants + variant('build_type', + default='Release', + description='CMake build type', + values=('Debug', 'Release', 'RelWithDebInfo', 'ASan')) + + variant('tests', + default=False, + description='Build and run tests for scord') + + variant("ofi", + default=True, + when="@0.2.0:", + description="Use OFI libfabric as transport library") + + variant("ucx", + default=False, + when="@0.2.0:", + description="Use UCX as transport library") + + + # general dependencies + depends_on("cmake@3.19", type='build') + + # specific dependencies + # v0.2.0+ + depends_on("argobots@1.1", when='@0.2.0:') + depends_on("mochi-margo@0.9.8", when='@0.2.0:') + depends_on("mochi-thallium@0.10.1", when='@0.2.0:') + depends_on("boost@1.71 +program_options", when='@0.2.0:') + depends_on("agios@latest", when='@0.2.0:') + depends_on("redis-plus-plus@1.3.3:", when='@0.2.0:') + + with when("@0.2.0: +ofi"): + depends_on("libfabric@1.14.0 fabrics=sockets,tcp,rxm") + depends_on("mercury@2.1.0 +ofi") + + with when("@0.2.0: +ucx"): + depends_on("ucx@1.12.0") + depends_on("mercury@2.1.0 +ucx") + + def cmake_args(self): + """Setup scord CMake arguments""" + cmake_args = [ + self.define_from_variant('SCORD_BUILD_TESTS', 'tests') + ] + return cmake_args + + def check(self): + """Run tests""" + with working_dir(self.build_directory): + make("test", parallel=False) diff --git a/spack/repo.yaml b/spack/repo.yaml new file mode 100644 index 00000000..f4fc33fb --- /dev/null +++ b/spack/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: 'admire' -- GitLab From 2a27a1ca9c1d31f76a59b0e982cafc6b5adb8be0 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 13 Feb 2023 10:15:01 +0100 Subject: [PATCH 2/2] Update README.md --- README.md | 146 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 124 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 83f0f518..9a6d004f 100644 --- a/README.md +++ b/README.md @@ -11,18 +11,120 @@ -## Software requirements +## Overview -Compiling and running `scord` requires up-to-date versions of various software -packages. Beware that using excessively old versions of these packages can +Scord is an open-source storage coordination service designed for Linux HPC +clusters. As a storage coordination service, Scord provides facilities for +system administrators to easily model their storage hierarchies and expose them +to users in a consistent manner. Once the storage architecture is modelled, +Scord can efficiently orchestrate data transfers between the different +storage tiers in order to support complex HPC workloads using node-local +storage, burst buffers and ad-hoc storage systems. + +> **ℹ️** **Important** +> This software was partially supported by the EuroHPC-funded project ADMIRE + (Project ID: 956748, https://www.admire-eurohpc.eu). + +## Building and installing + +This section describes how to build and install Scord. Two main options +are basically supported out of the box: + +- Automatically building both Scord and its dependencies using + [Spack](https://github.com/spack/spack). +- Build and install Scord manually. + +### Building scord and its dependencies with Spack + +Scord and all its dependencies can be built using +[Spack](https://github.com/spack/spack). If you already have Spack, make sure +you have the latest release. If you use a clone of the Spack `develop` +branch, be sure to pull the latest changes. + +#### Installing Spack + +If you haven't already, install Spack with the following commands: + +```shell +$ git clone -c feature.manyFiles=true https://github.com/spack/spack +``` + +This will create a directory called `spack` in your machine. Once you have +cloned Spack, we recommend sourcing the appropriate script for your shell. +This will add Spack to your `PATH` and enable the use of the `spack` command: + +```shell +# For bash/zsh/sh +$ . spack/share/spack/setup-env.sh + +# For tcsh/csh +$ source spack/share/spack/setup-env.csh + +# For fish +$ . spack/share/spack/setup-env.fish +``` + +Since Scord is not yet available in the official Spack repositories, you need +to add the Scord Spack repository to the local Spack namespace in your +machine. To do so, download the +[`spack/` directory](https://storage.bsc.es/gitlab/eu/admire/io-scheduler/-/tree/main/spack) +located in the root of Scord's repository root to your machine (e.g. under +`~/projects/scord/spack`) and execute the following: + +```shell +spack repo add ~/projects/scord/spack/ +``` + +You should now be able to fetch information from the Scord package using +Spack: + +```shell +spack info scord +``` + +If that worked, you are now ready to install Scord: + +```shell +spack install scord +``` + +You can include or remove variants with Spack when a custom Scord +build is desired. The available variants are listed below: + + +| Variant | Command | Default | Description | +|---------|-------------|---------|--------------------------------------------------------------------------| +| OFI | `scord+ofi` | True | Use [libfabric](https://github.com/ofiwg/libfabric) as transport library | +| UCX | `scord+ucx` | False | Use [ucx](https://github.com/openucx/ucx.git) as transport library | + + +> ⚠️ **Attention** +> The initial install could take a while as Spack will install build +> dependencies (autoconf, automake, m4, libtool, pkg-config, etc.) as well as +> any dependencies of dependencies (cmake, perl, etc.) if you don’t already +> have these dependencies installed through Spack. + +After the installation completes, remember that you first need to load +Scord in order to use it: + +```shell +spack load scord +``` + +### Building scord manually + +If you prefer to build and install Scord from sources, you can also do so +but bear in mind that compiling and running Scord requires up-to-date +versions of various software packages that will need to be available in your +system. Also beware that using excessively old versions of these packages can cause indirect errors that are very difficult to track down. -The following software packages are required to build `scord` and need to be -available in the system: +With that in mind, the following software packages are required to build Scord +and need to be available in the system: - A C++ compiler that supports the C++20 standard, for example [GCC](https://gcc.gnu.org)/[Clang](https://clang.llvm.org/) versions -8.0/5.0 or later. +8.0/6.0 or later. - [CMake](https://cmake.org) 3.19 or later. - pkg-config 0.29.1 (earlier versions will probably work but haven't been tested). - [yaml-cpp](https://github.com/jbeder/yaml-cpp) version 0.6.2 and later. @@ -40,7 +142,7 @@ available in the system: 3 or later, and its dependencies: - [hiredis](https://github.com/redis/hiredis) version 0.14.1 or later. -The following libraries are also required by `scord`, but will be +The following libraries are also required by Scord, but will be automatically downloaded and compiled by the project as part of the standard build process. @@ -53,16 +155,16 @@ build process. > **ℹ️** **Important** Margo and Argobots use `pkg-config` to ensure they compile and link correctly -with all of their dependencies' libraries. When building `scord` manually, +with all of their dependencies' libraries. When building Scord manually, you'll need to appropriately set either the ``PKG_CONFIG_PATH`` environment variable or the ``CMAKE_PREFIX_PATH`` CMake variable to the appropriate installation paths where the ``.pc`` files for Argobots and Margo reside. -## Building +#### Building You may either install the full sources or clone the repository directly. The package relies on CMake to build the service, which requires you to do an -out-of-source build. Thus, once the `scord` sources are located in an +out-of-source build. Thus, once the Scord sources are located in an appropriate `SCORD_SOURCES` directory (for example, `$HOME/scord`), you can build the service by running the following commands: @@ -77,9 +179,9 @@ cmake .. make ``` -### A more complex example: +#### A more complex example: -The following CMake options can be used to configure how `scord` is built: +The following CMake options can be used to configure how Scord is built: - `SCORD_TRANSPORT_LIBRARY`: This option allows configuring the transport library used by the service. Currently, both `libfabric` and `ucx` are @@ -101,18 +203,18 @@ The following CMake options can be used to configure how `scord` is built: - `SCORD_BUILD_TESTS`: This option instructs CMake to build the tests contained in the `tests` subdirectory. -Thus, let's assume that we want to build `scord` with the following +Thus, let's assume that we want to build Scord with the following configuration: 1. The service should use `libfabric` as the transport library. 2. The service should use `tcp` as the communication protocol. 3. The server should listen for RPCs on address `192.168.0.111` and on port `52000`. -5. Usage examples should be built. +4. Usage examples should be built. -Let's also assume that `scord` sources are located in `$HOME/scord`, that +Let's also assume that Scord sources are located in `$HOME/scord`, that dependencies were installed in `/opt`, and that we also want to install -`scord` in `/usr/local`. Taking into account all these requirements, `scord` +Scord in `/usr/local`. Taking into account all these requirements, Scord can be built by running the following commands: ```bash @@ -130,7 +232,7 @@ cmake -DCMAKE_PREFIX_PATH:STRING=/opt \ make ``` -## Running tests +#### Running tests Tests are integrated in [CTest](https://cmake.org/cmake/help/book/mastering-cmake/chapter/Testing%20With%20CMake%20and%20CTest.html), CMake's testing facility. Once built, the tests can be run in parallel using the `ctest` command line tool: @@ -146,10 +248,10 @@ Test project /home/amiranda/var/projects/scord/repo/build Total Test time (real) = 0.14 sec ``` -## Installing +#### Installing -Assuming that the CMAKE_INSTALL_PREFIX has been set (see previous step) and -that you have write permissions to the destination directory, `scord` can be +Assuming that the `CMAKE_INSTALL_PREFIX` has been set (see previous step) and +that you have write permissions to the destination directory, Scord can be installed by running the following command from the build directory: ```bash @@ -167,7 +269,7 @@ scord ## Testing the configuration If you just want to test that everything works as it should, you can start -`scord` in foreground mode and redirect its logging output to the console +Scord in foreground mode and redirect its logging output to the console with the following command: ```bash @@ -195,7 +297,7 @@ Which should produce output similar to the following: [2021-11-19 10:30:30.066151] [scord] [131119] [info] [2021-11-19 10:30:30.066161] [scord] [131119] [info] [[ Start up successful, awaiting requests... ]] ``` -Now we can use one of the example programs to send a `ping` RPC to `scord`: +Now we can use one of the example programs to send a `ping` RPC to Scord: ```bash -- GitLab