Commit bd0b9cc3 authored by Ramon Nou's avatar Ramon Nou
Browse files

Merge branch 'rnou/fallocate' into 'main' 

Fix fallocate error and add support for adhoc backends

fallocate is different from posix_fallocate, the first fails if the fallocate operation is not supported by the FS, like BEEGFS. However, the second one has a fail-safe and implements a write '0's to create the complete file.

In this MR , we also include support for different adhoc fs (parallel, posix, gekkofs, hercules, dataclay and expand)

We remove mpio in the ccp options and change it to parallel. Now is also posible to have a pure posix - posix transfer, which seems more performant than MPIIO

Closes #35, #36, #37

See merge request !25
parents b470a8e8 224fc9ba

CMakeLists.txt

+21 −0
Original line number Diff line number Diff line
@@ -260,6 +260,27 @@ FetchContent_Declare( 


FetchContent_MakeAvailable(expected)



### GekkoFS: Optional for gekkofs

find_package(GekkoFS)

if (GekkoFS_FOUND)

  add_compile_definitions(GEKKOFS_PLUGIN)

  message(STATUS "[${PROJECT_NAME}] Found GekkoFS")

endif()



### Hercules: Optional for hercules

find_package(Hercules)

if (Hercules_FOUND)

  add_compile_definitions(HERCULES_PLUGIN)

  message(STATUS "[${PROJECT_NAME}] Found Hercules")

endif()



### Expand: Optional for expand

find_package(Expand)

if (Expand_FOUND)

  add_compile_definitions(EXPAND_PLUGIN)

  message(STATUS "[${PROJECT_NAME}] Found Expand")

endif()



### Threads: required by ASIO

find_package(Threads REQUIRED)

cli/copy.cpp

+17 −14
Original line number Diff line number Diff line
@@ -29,19 +29,24 @@ 
#include <CLI/CLI.hpp>

#include <ranges>



enum class dataset_flags { posix, mpio };

enum class dataset_flags { posix, parallel, none, gekkofs, hercules, expand, dataclay };



std::map<std::string, dataset_flags> dataset_flags_map{

        {"posix", dataset_flags::posix},

        {"mpio", dataset_flags::mpio}};

std::map<std::string, cargo::dataset::type> dataset_flags_map{

        {"posix", cargo::dataset::type::posix},

        {"parallel", cargo::dataset::type::parallel},

        {"none", cargo::dataset::type::none},

        {"gekkofs", cargo::dataset::type::gekkofs},

        {"hercules", cargo::dataset::type::hercules},

        {"expand", cargo::dataset::type::expand},

        {"dataclay", cargo::dataset::type::dataclay}};



struct copy_config {

    std::string progname;

    std::string server_address;

    std::vector<std::filesystem::path> inputs;

    dataset_flags input_flags = dataset_flags::posix;

    cargo::dataset::type input_flags = cargo::dataset::type::posix;

    std::vector<std::filesystem::path> outputs;

    dataset_flags output_flags = dataset_flags::posix;

    cargo::dataset::type output_flags = cargo::dataset::type::posix;

};



copy_config
@@ -72,7 +77,8 @@ parse_command_line(int argc, char* argv[]) { 
    app.add_option("--if", cfg.input_flags,

                   "Flags for input datasets. Accepted values\n"

                   "  - posix: read data using POSIX (default)\n"

                   "  - mpio: read data using MPI-IO")

                   "  - parallel: read data using MPI-IO\n"

                   "  - gekkofs: read data using gekkofs user library\n")

            ->option_text("FLAGS")

            ->transform(CLI::CheckedTransformer(dataset_flags_map,

                                                CLI::ignore_case));
@@ -80,7 +86,8 @@ parse_command_line(int argc, char* argv[]) { 
    app.add_option("--of", cfg.output_flags,

                   "Flags for output datasets. Accepted values\n"

                   "  - posix: write data using POSIX (default)\n"

                   "  - mpio: write data using MPI-IO")

                   "  - parallel: write data using MPI-IO\n"

                   "  - gekkofs: write data using gekkofs user library\n")

            ->option_text("FLAGS")

            ->transform(CLI::CheckedTransformer(dataset_flags_map,

                                                CLI::ignore_case));
@@ -119,17 +126,13 @@ main(int argc, char* argv[]) { 
        std::transform(cfg.inputs.cbegin(), cfg.inputs.cend(),

                       std::back_inserter(inputs), [&](const auto& src) {

                           return cargo::dataset{

                                   src, cfg.input_flags == dataset_flags::mpio

                                                ? cargo::dataset::type::parallel

                                                : cargo::dataset::type::posix};

                                   src, cfg.input_flags};

                       });



        std::transform(cfg.outputs.cbegin(), cfg.outputs.cend(),

                       std::back_inserter(outputs), [&cfg](const auto& tgt) {

                           return cargo::dataset{

                                   tgt, cfg.output_flags == dataset_flags::mpio

                                                ? cargo::dataset::type::parallel

                                                : cargo::dataset::type::posix};

                                   tgt, cfg.output_flags};

                       });



        const auto tx = cargo::transfer_datasets(server, inputs, outputs);

cmake/FindExpand.cmake

0 → 100644
+65 −0
Original line number Diff line number Diff line 
################################################################################

# Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany          #

#                                                                              #

# This software was partially supported by the                                 #

# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #

#                                                                              #

# This software was partially supported by the                                 #

# ADA-FS project under the SPPEXA project funded by the DFG.                   #

#                                                                              #

# This file is part of GekkoFS.                                                #

#                                                                              #

# GekkoFS is free software: you can redistribute it and/or modify              #

# it under the terms of the GNU General Public License as published by         #

# the Free Software Foundation, either version 3 of the License, or            #

# (at your option) any later version.                                          #

#                                                                              #

# GekkoFS is distributed in the hope that it will be useful,                   #

# but WITHOUT ANY WARRANTY; without even the implied warranty of               #

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #

# GNU General Public License for more details.                                 #

#                                                                              #

# You should have received a copy of the GNU General Public License            #

# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #

#                                                                              #

# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################





find_path(Expand_INCLUDE_DIR

  NAMES user_functions.hpp

  PREFIX gkfs

)



find_library(Expand_LIBRARY

  NAMES libexpand_user_lib.so

)



include(FindPackageHandleStandardArgs)

find_package_handle_standard_args(

	Expand

	DEFAULT_MSG

	Expand_INCLUDE_DIR

	Expand_LIBRARY

)



if(Expand_FOUND)

  set(Expand_LIBRARIES ${Expand_LIBRARY})

  set(Expand_INCLUDE_DIRS ${Expand_INCLUDE_DIR})





  if(NOT TARGET Expand::Expand)

	  add_library(Expand::Expand UNKNOWN IMPORTED)

	  set_target_properties(Expand::Expand PROPERTIES

		IMPORTED_LOCATION "${Expand_LIBRARY}"

		INTERFACE_INCLUDE_DIRECTORIES "${Expand_INCLUDE_DIR}"

	  )

	endif()

endif()





mark_as_advanced(

	Expand_INCLUDE_DIR

	Expand_LIBRARY

)

cmake/FindGekkoFS.cmake

0 → 100644
+65 −0
Original line number Diff line number Diff line 
################################################################################

# Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany          #

#                                                                              #

# This software was partially supported by the                                 #

# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #

#                                                                              #

# This software was partially supported by the                                 #

# ADA-FS project under the SPPEXA project funded by the DFG.                   #

#                                                                              #

# This file is part of GekkoFS.                                                #

#                                                                              #

# GekkoFS is free software: you can redistribute it and/or modify              #

# it under the terms of the GNU General Public License as published by         #

# the Free Software Foundation, either version 3 of the License, or            #

# (at your option) any later version.                                          #

#                                                                              #

# GekkoFS is distributed in the hope that it will be useful,                   #

# but WITHOUT ANY WARRANTY; without even the implied warranty of               #

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #

# GNU General Public License for more details.                                 #

#                                                                              #

# You should have received a copy of the GNU General Public License            #

# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #

#                                                                              #

# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################





find_path(GekkoFS_INCLUDE_DIR

  NAMES user_functions.hpp

  PREFIX gkfs

)



find_library(GekkoFS_LIBRARY

  NAMES libgkfs_user_lib.so

)



include(FindPackageHandleStandardArgs)

find_package_handle_standard_args(

	GekkoFS

	DEFAULT_MSG

	GekkoFS_INCLUDE_DIR

	GekkoFS_LIBRARY

)



if(GekkoFS_FOUND)

  set(GekkoFS_LIBRARIES ${GekkoFS_LIBRARY})

  set(GekkoFS_INCLUDE_DIRS ${GekkoFS_INCLUDE_DIR})





  if(NOT TARGET GekkoFS::GekkoFS)

	  add_library(GekkoFS::GekkoFS UNKNOWN IMPORTED)

	  set_target_properties(GekkoFS::GekkoFS PROPERTIES

		IMPORTED_LOCATION "${GekkoFS_LIBRARY}"

		INTERFACE_INCLUDE_DIRECTORIES "${GekkoFS_INCLUDE_DIR}"

	  )

	endif()

endif()





mark_as_advanced(

	GekkoFS_INCLUDE_DIR

	GekkoFS_LIBRARY

)

cmake/FindHercules.cmake

0 → 100644
+65 −0
Original line number Diff line number Diff line 
################################################################################

# Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany          #

#                                                                              #

# This software was partially supported by the                                 #

# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #

#                                                                              #

# This software was partially supported by the                                 #

# ADA-FS project under the SPPEXA project funded by the DFG.                   #

#                                                                              #

# This file is part of GekkoFS.                                                #

#                                                                              #

# GekkoFS is free software: you can redistribute it and/or modify              #

# it under the terms of the GNU General Public License as published by         #

# the Free Software Foundation, either version 3 of the License, or            #

# (at your option) any later version.                                          #

#                                                                              #

# GekkoFS is distributed in the hope that it will be useful,                   #

# but WITHOUT ANY WARRANTY; without even the implied warranty of               #

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #

# GNU General Public License for more details.                                 #

#                                                                              #

# You should have received a copy of the GNU General Public License            #

# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #

#                                                                              #

# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################





find_path(Hercules_INCLUDE_DIR

  NAMES user_functions.hpp

  PREFIX hercules

)



find_library(Hercules_LIBRARY

  NAMES libhercules_user_lib.so

)



include(FindPackageHandleStandardArgs)

find_package_handle_standard_args(

	Hercules

	DEFAULT_MSG

	Hercules_INCLUDE_DIR

	Hercules_LIBRARY

)



if(Hercules_FOUND)

  set(Hercules_LIBRARIES ${Hercules_LIBRARY})

  set(Hercules_INCLUDE_DIRS ${Hercules_INCLUDE_DIR})





  if(NOT TARGET Hercules::Hercules)

	  add_library(Hercules::Hercules UNKNOWN IMPORTED)

	  set_target_properties(Hercules::Hercules PROPERTIES

		IMPORTED_LOCATION "${Hercules_LIBRARY}"

		INTERFACE_INCLUDE_DIRECTORIES "${Hercules_INCLUDE_DIR}"

	  )

	endif()

endif()





mark_as_advanced(

	Hercules_INCLUDE_DIR

	Hercules_LIBRARY

)