Resolve "Add a user level library for functionalities not requiring direct `scord` intervention"

Alberto Mirandarequested to merge
amiranda/119-add-a-user-level-library-for-functionalities-not-requiring-direct-scord-intervention into main

This MR adds the library libscord-user.so with user-level interfaces to request data transfers from scord from application code.

Closes #119 (closed)

Edited by Alberto Miranda

Merge request reports

examples/c/ADM_transfer_datasets_user.c

0 → 100644
+104 −0
Changes for examples/c/ADM_transfer_datasets_user.c: 104 added lines, 0 removed lines.
Original line number Diff line number Diff line 
/******************************************************************************

 * Copyright 2021-2023, Barcelona Supercomputing Center (BSC), Spain

 *

 * This software was partially supported by the EuroHPC-funded project ADMIRE

 *   (Project ID: 956748, https://www.admire-eurohpc.eu).

 *

 * This file is part of scord.

 *

 * scord is free software: you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation, either version 3 of the License, or

 * (at your option) any later version.

 *

 * scord is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with scord.  If not, see <https://www.gnu.org/licenses/>.

 *

 * SPDX-License-Identifier: GPL-3.0-or-later

 *****************************************************************************/



#include <scord/scord-user.h>

#include <stdio.h>

#include "common.h"



int

main(int argc, char** argv) {



    enum { N = 5 };



    test_info_t test_info = {

            .name = TESTNAME,

            .requires_server = true,

            .requires_controller = true,

    };



    cli_args_t cli_args;

    if(process_args(argc, argv, test_info, &cli_args)) {

        exit(EXIT_FAILURE);

    }



    const char* paths[N] = {"input00.dat", "inpu01.dat", "input02.dat",

                            "input03.dat", "input04.dat"};



    ADM_dataset_t sources[N];

    ADM_dataset_t targets[N];



    for(size_t i = 0; i < N; ++i) {

        sources[i] = ADM_dataset_create(/*"lustre", */ paths[i]);

        targets[i] = ADM_dataset_create(/*"gekkofs",*/ paths[i]);



        if(!sources[i] || !targets[i]) {

            abort();

        }

    }



    ADM_return_t ret = ADM_SUCCESS;

    ADM_transfer_t tx;



    // the library will automatically route the request to the `scord`

    // server configured in the cluster

    if((ret = ADM_transfer_datasets(sources, N, targets, N, &tx)) !=

       ADM_SUCCESS) {

        fprintf(stderr, "ADM_transfer_datasets() failed: %s\n",

                ADM_strerror(ret));

        abort();

    }



    ADM_transfer_status_t status;

    do {

        struct timespec timeout = {.tv_sec = 1, .tv_nsec = 0};

        // Wait for the transfer to complete

        ret = ADM_transfer_wait(tx, &status, &timeout);



        if(ret != ADM_SUCCESS && ret != ADM_ETIMEOUT) {

            fprintf(stderr, "ADM_transfer_wait() failed: %s\n",

                    ADM_strerror(ret));

            abort();

        }



        if(ADM_TRANSFER_SUCCEEDED(status)) {

            fprintf(stdout, "Transfer completed successfully\n");

        } else if(ADM_TRANSFER_FAILED(status)) {

            fprintf(stderr, "Transfer failed: %s\n",

                    ADM_strerror(ADM_TRANSFER_ERROR(status)));

        } else if(ADM_TRANSFER_PENDING(status)) {

            fprintf(stdout, "Transfer pending\n");

            continue;

        } else if(ADM_TRANSFER_IN_PROGRESS(status)) {

            fprintf(stdout, "Transfer in progress\n");

            continue;

        } else {

            fprintf(stderr, "Transfer status unknown\n");

            abort();

        }

    } while(!ADM_TRANSFER_SUCCEEDED(status));



    free(status);



    return 0;

}

examples/c/CMakeLists.txt

+17 −1
Changes for examples/c/CMakeLists.txt: 17 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -31,6 +31,8 @@ list(APPEND c_examples_with_controller 
  # transfers

  ADM_transfer_datasets ADM_get_transfer_priority ADM_set_transfer_priority

  ADM_cancel_transfer ADM_get_pending_transfers

  # transfers (user)

  ADM_transfer_datasets_user

  # qos

  ADM_set_qos_constraints ADM_get_qos_constraints

  # data operations
@@ -55,7 +57,15 @@ target_link_libraries(c_examples_common libscord_c_types) 
foreach(example IN LISTS c_examples_with_controller c_examples_without_controller)

  add_executable(${example}_c)

  target_sources(${example}_c PRIVATE ${example}.c)

  target_link_libraries(${example}_c PUBLIC libscord c_examples_common)



  if (example MATCHES "^.*_user$")

    target_link_libraries(${example}_c PUBLIC libscord-user)

  else ()

    target_link_libraries(${example}_c PUBLIC libscord)

  endif ()



  target_link_libraries(${example}_c PUBLIC c_examples_common)



  set_target_properties(${example}_c PROPERTIES OUTPUT_NAME ${example})

endforeach()
@@ -92,6 +102,12 @@ if(SCORD_BUILD_TESTS) 
      )

  endforeach()



  # Disable until we have more than stubs for the user library

  set_tests_properties(run_ADM_transfer_datasets_user_c_test

    PROPERTIES DISABLED TRUE)

  set_tests_properties(validate_ADM_transfer_datasets_user_c_test

    PROPERTIES DISABLED TRUE)



  foreach(example IN LISTS c_examples_without_controller)



    # prepare environment for the RPC test itself and its validation test

src/lib/scord/scord-user.h

0 → 100644
+127 −0
Changes for src/lib/scord/scord-user.h: 127 added lines, 0 removed lines.
Original line number Diff line number Diff line 
/******************************************************************************

 * Copyright 2021-2023, Barcelona Supercomputing Center (BSC), Spain

 *

 * This software was partially supported by the EuroHPC-funded project ADMIRE

 *   (Project ID: 956748, https://www.admire-eurohpc.eu).

 *

 * This file is part of scord.

 *

 * scord is free software: you can redistribute it and/or modify

 * it under the terms of the Lesser GNU General Public License as published by

 * the Free Software Foundation, either version 3 of the License, or

 * (at your option) any later version.

 *

 * scord is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the Lesser GNU General Public License

 * along with scord.  If not, see <https://www.gnu.org/licenses/>.

 *

 * SPDX-License-Identifier: LGPL-3.0-or-later

 *****************************************************************************/





#ifndef SCORD_USER_H

#define SCORD_USER_H



#include <time.h>

#include <scord/types.h>



typedef struct adm_transfer_status* ADM_transfer_status_t;



#ifdef __cplusplus

extern "C" {

#endif



/**

 * @brief Transfer datasets between storage tiers asynchronously.

 *

 * @param[in] sources The datasets to transfer.

 * @param[in] sources_len The number of datasets to transfer.

 * @param[in] targets The destination datasets.

 * @param[in] targets_len The number of destination datasets.

 * @param[out] transfer A transfer handle to query the operation status.

 * @return

 */

ADM_return_t

ADM_transfer_datasets(ADM_dataset_t sources[], size_t sources_len,

                      ADM_dataset_t targets[], size_t targets_len,

                      ADM_transfer_t* transfer);





/**

 * @brief Wait for a transfer to complete.

 *

 * @param[in] transfer The transfer handle.

 * @param[out] status A pointer to a dynamically allocated

 * `ADM_transfer_status_t` structure where status information should be stored.

 * The caller is responsible for freeing the memory allocated for this

 * structure.

 * @param[out] timeout The maximum time to wait for the transfer to complete. If

 * NULL, query the transfer status and return immediately. If not NULL, wait for

 * the transfer to complete or the timeout to expire.

 * @return ADM_SUCCESS if the transfer completed successfully. ADM_ETIMEOUT if

 * the transfer did not complete before the timeout expired. An

 * specifc ADM_E* error code otherwise.

 */

ADM_return_t

ADM_transfer_wait(ADM_transfer_t transfer, ADM_transfer_status_t* status,

                  struct timespec* timeout);



bool

__adm_transfer_succeeded(ADM_transfer_status_t status);



bool

__adm_transfer_failed(ADM_transfer_status_t status);



bool

__adm_transfer_pending(ADM_transfer_status_t status);



bool

__adm_transfer_in_progress(ADM_transfer_status_t status);



ADM_return_t

__adm_transfer_error(ADM_transfer_status_t status);



/**

 * @brief Check if a transfer completed successfully.

 */

#define ADM_TRANSFER_SUCCEEDED(st) __adm_transfer_succeeded(st)



/**

 * @brief Check if a transfer failed.

 */

#define ADM_TRANSFER_FAILED(st) __adm_transfer_failed(st)



/**

 * @brief Check if a transfer is pending.

 */

#define ADM_TRANSFER_PENDING(st) __adm_transfer_pending(st)



/**

 * @brief Check if a transfer is in progress.

 */

#define ADM_TRANSFER_IN_PROGRESS(st) __adm_transfer_in_progress(st)



/**

 * @brief Return the error code of a failed transfer.

 */

#define ADM_TRANSFER_ERROR(st) __adm_transfer_error(st)



/**

 * Return a string describing the error number

 *

 * @param[in] errnum The error number for which a description should be

 * returned.

 * @return A pointer to a string describing `errnum`.

 */

const char*

ADM_strerror(ADM_return_t errnum);



#ifdef __cplusplus

} // extern "C"

#endif



#endif // SCORD_USER_H

src/lib/scord/types.h

+17 −4
Changes for src/lib/scord/types.h: 17 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -25,15 +25,15 @@ 
#ifndef SCORD_TYPES_H

#define SCORD_TYPES_H



#ifdef __cplusplus

extern "C" {

#endif



#include <stdbool.h>

#include <stddef.h>

#include <stdint.h>

#include <stdlib.h>



#ifdef __cplusplus

extern "C" {

#endif



/******************************************************************************/

/* Public type and struct definitions                                         */

/******************************************************************************/
@@ -53,6 +53,7 @@ typedef enum { 
    ADM_EADHOC_DIR_CREATE_FAILED,

    ADM_EADHOC_DIR_EXISTS,

    ADM_ESUBPROCESS_ERROR,

    ADM_ETIMEOUT,

    ADM_EOTHER,

    ADM_ERR_MAX = 512

} ADM_return_t;
@@ -201,9 +202,21 @@ typedef enum { 
    ADM_MAPPING_N_TO_N

} ADM_transfer_mapping_t;



/** A transfer state */

typedef enum {

    ADM_TRANSFER_QUEUED,

    ADM_TRANSFER_RUNNING,

    ADM_TRANSFER_FINISHED,

    ADM_TRANSFER_FAILED,

    ADM_TRANSFER_CANCELLED

} ADM_transfer_state_t;



/** A handle to a created transfer */

typedef struct adm_transfer* ADM_transfer_t;



/** A transfer status */

typedef struct adm_transfer_status* ADM_transfer_status_t;



/** A transfer priority */

typedef int ADM_transfer_priority_t;

src/lib/CMakeLists.txt

+35 −2
Changes for src/lib/CMakeLists.txt: 35 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -79,7 +79,7 @@ target_link_libraries( 
set_property(TARGET libscord_cxx_types PROPERTY POSITION_INDEPENDENT_CODE ON)



################################################################################

# Create a target for the actual library that will be used by clients

# Create a target for the actual library that will be used by admin clients

################################################################################

add_library(libscord SHARED)
@@ -107,8 +107,41 @@ target_link_libraries( 


set_target_properties(libscord PROPERTIES OUTPUT_NAME "scord")



################################################################################

# Create a target for the actual library that will be used by users

################################################################################

add_library(libscord-user SHARED)



target_sources(

  libscord-user

  PUBLIC scord/scord-user.h

  PRIVATE libscord-user.c

)



set(public_headers, "")

list(APPEND public_headers "scord/scord-user.h")



set_target_properties(libscord-user PROPERTIES PUBLIC_HEADER

  "${public_headers}")



target_include_directories(

  libscord-user PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>

  $<INSTALL_INTERFACE:include/${PROJECT_NAME}>

)



target_link_libraries(

  libscord-user

  PRIVATE common::network::rpc_client

  PUBLIC libscord_c_types

)



set_target_properties(libscord-user PROPERTIES OUTPUT_NAME "scord-user")



################################################################################

# Install the libraries

################################################################################

install(

  TARGETS libscord libscord_c_types libscord_cxx_types

  TARGETS libscord libscord-user libscord_c_types libscord_cxx_types

  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}

  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}

  PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}