Resolve "Adhoc storage deployment needs to be refactored"

Alberto Mirandarequested to merge
amiranda/116-adhoc-storage-deployment-needs-to-be-refactored into main

This MR implements a generic mechanism for deploying adhoc storage systems in scord-ctl. The mechanism allows users to define via the scord-ctl.conf configuration file which scripts should be used to start and stop an adhoc storage system.

Closes #116 (closed)

Edited by Alberto Miranda

Merge request reports

etc/deploy_scripts/CMakeLists.txt

0 → 100644
+35 −0
Changes for etc/deploy_scripts/CMakeLists.txt: 35 added lines, 0 removed lines.
Original line number Diff line number Diff line 
################################################################################

# Copyright 2021-2023, Barcelona Supercomputing Center (BSC), Spain            #

#                                                                              #

# This software was partially supported by the EuroHPC-funded project ADMIRE   #

#   (Project ID: 956748, https://www.admire-eurohpc.eu).                       #

#                                                                              #

# This file is part of scord.                                                  #

#                                                                              #

# scord is free software: you can redistribute it and/or modify                #

# it under the terms of the GNU General Public License as published by         #

# the Free Software Foundation, either version 3 of the License, or            #

# (at your option) any later version.                                          #

#                                                                              #

# scord is distributed in the hope that it will be useful,                     #

# but WITHOUT ANY WARRANTY; without even the implied warranty of               #

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #

# GNU General Public License for more details.                                 #

#                                                                              #

# You should have received a copy of the GNU General Public License            #

# along with scord.  If not, see <https://www.gnu.org/licenses/>.              #

#                                                                              #

# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################



list(APPEND ADHOC_SCRIPTS "${CMAKE_CURRENT_SOURCE_DIR}/gekkofs.sh")



# install adhoc scripts to `<sysconfdir>/scord` (normally /etc/scord)

install(

  FILES ${ADHOC_SCRIPTS}

  DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/${PROJECT_NAME}

  PERMISSIONS

    OWNER_EXECUTE OWNER_WRITE OWNER_READ

    GROUP_EXECUTE GROUP_READ

    WORLD_EXECUTE WORLD_READ

)

etc/deploy_scripts/gekkofs.sh

0 → 100644
+3 −0
Changes for etc/deploy_scripts/gekkofs.sh: 3 added lines, 0 removed lines.
Original line number Diff line number Diff line 
#!/usr/bin/env bash



exit 0

etc/CMakeLists.txt

+4 −0
Changes for etc/CMakeLists.txt: 4 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -22,10 +22,14 @@ 
# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################



add_subdirectory(deploy_scripts)



configure_file(scord.conf.in scord.conf)

configure_file(scord-ctl.conf.in scord-ctl.conf @ONLY)



# install the configuration file to sysconfdir (normally /etc)

install(FILES ${CMAKE_CURRENT_BINARY_DIR}/scord.conf

              ${CMAKE_CURRENT_BINARY_DIR}/scord-ctl.conf

        DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}

)

etc/scord-ctl.conf.in

0 → 100644
+48 −0
Changes for etc/scord-ctl.conf.in: 48 added lines, 0 removed lines.
Original line number Diff line number Diff line 
## vim: set filetype=yaml:



# Configuration of the `scord-ctl` service

config:

  # Specific configurations for supported adhoc storage systems

  adhoc_storage:

    gekkofs:

      # The default working directory for adhoc instances of this type

      working_directory: /tmp/gekkofs

      startup:

        # Specific environment variables that should be set for the adhoc

        # instance. These will be merged with the environment variables

        # already set by Slurm.

        environment:

          VAR0: value0

          VAR1: value1

        # The command that scord-ctl will use to start an adhoc instance of

        # this type. The following variables are supported that will be

        # automatically replaced by scord-ctl if found between curly braces:

        #  * ADHOC_NODES: A comma separated list of valid job hostnames that

        #    can be used to start the adhoc instance.

        #  * ADHOC_DIRECTORY: A unique working directory for each specific

        #    adhoc instance. This directory will be created by scord-ctl under

        #    `working_directory` and automatically removed after the adhoc

        #    instance has been shut down.

        #  * ADHOC_ID: - A unique ID for the adhoc instance.

        command: @CMAKE_INSTALL_FULL_SYSCONFDIR@/@PROJECT_NAME@/gekkofs.sh

                    start

                    --hosts {ADHOC_NODES}

                    --workdir {ADHOC_DIRECTORY}

                    --datadir {ADHOC_DIRECTORY}/data

                    --mountdir {ADHOC_DIRECTORY}/mnt

      shutdown:

        environment:

        command: @CMAKE_INSTALL_FULL_SYSCONFDIR@/@PROJECT_NAME@/gekkofs.sh

                    stop

                    --workdir {ADHOC_DIRECTORY}





# default storage tiers made available to applications

storage:

  lustre:

    type: "pfs"

    mountpoint: "/mnt/lustre"



  tmp:

    type: "tmpfs"

    mountpoint: "/tmp"

src/common/net/server.cpp

+2 −2
Changes for src/common/net/server.cpp: 2 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -263,7 +263,7 @@ server::install_signal_handlers() { 
}



void

server::check_configuration() {}

server::check_configuration() const {}



void

server::print_greeting() {
@@ -276,7 +276,7 @@ server::print_greeting() { 
}



void

server::print_configuration() {

server::print_configuration() const {

    LOGGER_INFO("");

    LOGGER_INFO("[[ Configuration ]]");

    LOGGER_INFO("  - running as daemon?: {}", m_daemonize ? "yes" : "no");