Fix fallocate error and add support for adhoc backends

Ramon Nourequested to merge
rnou/fallocate into main

fallocate is different from posix_fallocate, the first fails if the fallocate operation is not supported by the FS, like BEEGFS. However, the second one has a fail-safe and implements a write '0's to create the complete file.

In this MR , we also include support for different adhoc fs (parallel, posix, gekkofs, hercules, dataclay and expand)

We remove mpio in the ccp options and change it to parallel. Now is also posible to have a pure posix - posix transfer, which seems more performant than MPIIO

Closes #35 (closed), #36 (closed), #37 (closed)

Merge request reports

cli/copy.cpp

+17 −14
Changes for cli/copy.cpp: 17 added lines, 14 removed lines.
Original line number Diff line number Diff line
@@ -29,19 +29,24 @@ 
#include <CLI/CLI.hpp>

#include <ranges>



enum class dataset_flags { posix, mpio };

enum class dataset_flags { posix, parallel, none, gekkofs, hercules, expand, dataclay };



std::map<std::string, dataset_flags> dataset_flags_map{

        {"posix", dataset_flags::posix},

        {"mpio", dataset_flags::mpio}};

std::map<std::string, cargo::dataset::type> dataset_flags_map{

        {"posix", cargo::dataset::type::posix},

        {"parallel", cargo::dataset::type::parallel},

        {"none", cargo::dataset::type::none},

        {"gekkofs", cargo::dataset::type::gekkofs},

        {"hercules", cargo::dataset::type::hercules},

        {"expand", cargo::dataset::type::expand},

        {"dataclay", cargo::dataset::type::dataclay}};



struct copy_config {

    std::string progname;

    std::string server_address;

    std::vector<std::filesystem::path> inputs;

    dataset_flags input_flags = dataset_flags::posix;

    cargo::dataset::type input_flags = cargo::dataset::type::posix;

    std::vector<std::filesystem::path> outputs;

    dataset_flags output_flags = dataset_flags::posix;

    cargo::dataset::type output_flags = cargo::dataset::type::posix;

};



copy_config
@@ -72,7 +77,8 @@ parse_command_line(int argc, char* argv[]) { 
    app.add_option("--if", cfg.input_flags,

                   "Flags for input datasets. Accepted values\n"

                   "  - posix: read data using POSIX (default)\n"

                   "  - mpio: read data using MPI-IO")

                   "  - parallel: read data using MPI-IO\n"

                   "  - gekkofs: read data using gekkofs user library\n")

            ->option_text("FLAGS")

            ->transform(CLI::CheckedTransformer(dataset_flags_map,

                                                CLI::ignore_case));
@@ -80,7 +86,8 @@ parse_command_line(int argc, char* argv[]) { 
    app.add_option("--of", cfg.output_flags,

                   "Flags for output datasets. Accepted values\n"

                   "  - posix: write data using POSIX (default)\n"

                   "  - mpio: write data using MPI-IO")

                   "  - parallel: write data using MPI-IO\n"

                   "  - gekkofs: write data using gekkofs user library\n")

            ->option_text("FLAGS")

            ->transform(CLI::CheckedTransformer(dataset_flags_map,

                                                CLI::ignore_case));
@@ -119,17 +126,13 @@ main(int argc, char* argv[]) { 
        std::transform(cfg.inputs.cbegin(), cfg.inputs.cend(),

                       std::back_inserter(inputs), [&](const auto& src) {

                           return cargo::dataset{

                                   src, cfg.input_flags == dataset_flags::mpio

                                                ? cargo::dataset::type::parallel

                                                : cargo::dataset::type::posix};

                                   src, cfg.input_flags};

                       });



        std::transform(cfg.outputs.cbegin(), cfg.outputs.cend(),

                       std::back_inserter(outputs), [&cfg](const auto& tgt) {

                           return cargo::dataset{

                                   tgt, cfg.output_flags == dataset_flags::mpio

                                                ? cargo::dataset::type::parallel

                                                : cargo::dataset::type::posix};

                                   tgt, cfg.output_flags};

                       });



        const auto tx = cargo::transfer_datasets(server, inputs, outputs);

cmake/FindExpand.cmake

0 → 100644
+65 −0
Changes for cmake/FindExpand.cmake: 65 added lines, 0 removed lines.
Original line number Diff line number Diff line 
################################################################################

# Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany          #

#                                                                              #

# This software was partially supported by the                                 #

# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #

#                                                                              #

# This software was partially supported by the                                 #

# ADA-FS project under the SPPEXA project funded by the DFG.                   #

#                                                                              #

# This file is part of GekkoFS.                                                #

#                                                                              #

# GekkoFS is free software: you can redistribute it and/or modify              #

# it under the terms of the GNU General Public License as published by         #

# the Free Software Foundation, either version 3 of the License, or            #

# (at your option) any later version.                                          #

#                                                                              #

# GekkoFS is distributed in the hope that it will be useful,                   #

# but WITHOUT ANY WARRANTY; without even the implied warranty of               #

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #

# GNU General Public License for more details.                                 #

#                                                                              #

# You should have received a copy of the GNU General Public License            #

# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #

#                                                                              #

# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################





find_path(Expand_INCLUDE_DIR

  NAMES user_functions.hpp

  PREFIX gkfs

)



find_library(Expand_LIBRARY

  NAMES libexpand_user_lib.so

)



include(FindPackageHandleStandardArgs)

find_package_handle_standard_args(

	Expand

	DEFAULT_MSG

	Expand_INCLUDE_DIR

	Expand_LIBRARY

)



if(Expand_FOUND)

  set(Expand_LIBRARIES ${Expand_LIBRARY})

  set(Expand_INCLUDE_DIRS ${Expand_INCLUDE_DIR})





  if(NOT TARGET Expand::Expand)

	  add_library(Expand::Expand UNKNOWN IMPORTED)

	  set_target_properties(Expand::Expand PROPERTIES

		IMPORTED_LOCATION "${Expand_LIBRARY}"

		INTERFACE_INCLUDE_DIRECTORIES "${Expand_INCLUDE_DIR}"

	  )

	endif()

endif()





mark_as_advanced(

	Expand_INCLUDE_DIR

	Expand_LIBRARY

)

cmake/FindGekkoFS.cmake

0 → 100644
+65 −0
Changes for cmake/FindGekkoFS.cmake: 65 added lines, 0 removed lines.
Original line number Diff line number Diff line 
################################################################################

# Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany          #

#                                                                              #

# This software was partially supported by the                                 #

# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #

#                                                                              #

# This software was partially supported by the                                 #

# ADA-FS project under the SPPEXA project funded by the DFG.                   #

#                                                                              #

# This file is part of GekkoFS.                                                #

#                                                                              #

# GekkoFS is free software: you can redistribute it and/or modify              #

# it under the terms of the GNU General Public License as published by         #

# the Free Software Foundation, either version 3 of the License, or            #

# (at your option) any later version.                                          #

#                                                                              #

# GekkoFS is distributed in the hope that it will be useful,                   #

# but WITHOUT ANY WARRANTY; without even the implied warranty of               #

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #

# GNU General Public License for more details.                                 #

#                                                                              #

# You should have received a copy of the GNU General Public License            #

# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #

#                                                                              #

# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################





find_path(GekkoFS_INCLUDE_DIR

  NAMES user_functions.hpp

  PREFIX gkfs

)



find_library(GekkoFS_LIBRARY

  NAMES libgkfs_user_lib.so

)



include(FindPackageHandleStandardArgs)

find_package_handle_standard_args(

	GekkoFS

	DEFAULT_MSG

	GekkoFS_INCLUDE_DIR

	GekkoFS_LIBRARY

)



if(GekkoFS_FOUND)

  set(GekkoFS_LIBRARIES ${GekkoFS_LIBRARY})

  set(GekkoFS_INCLUDE_DIRS ${GekkoFS_INCLUDE_DIR})





  if(NOT TARGET GekkoFS::GekkoFS)

	  add_library(GekkoFS::GekkoFS UNKNOWN IMPORTED)

	  set_target_properties(GekkoFS::GekkoFS PROPERTIES

		IMPORTED_LOCATION "${GekkoFS_LIBRARY}"

		INTERFACE_INCLUDE_DIRECTORIES "${GekkoFS_INCLUDE_DIR}"

	  )

	endif()

endif()





mark_as_advanced(

	GekkoFS_INCLUDE_DIR

	GekkoFS_LIBRARY

)

cmake/FindHercules.cmake

0 → 100644
+65 −0
Changes for cmake/FindHercules.cmake: 65 added lines, 0 removed lines.
Original line number Diff line number Diff line 
################################################################################

# Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany          #

#                                                                              #

# This software was partially supported by the                                 #

# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #

#                                                                              #

# This software was partially supported by the                                 #

# ADA-FS project under the SPPEXA project funded by the DFG.                   #

#                                                                              #

# This file is part of GekkoFS.                                                #

#                                                                              #

# GekkoFS is free software: you can redistribute it and/or modify              #

# it under the terms of the GNU General Public License as published by         #

# the Free Software Foundation, either version 3 of the License, or            #

# (at your option) any later version.                                          #

#                                                                              #

# GekkoFS is distributed in the hope that it will be useful,                   #

# but WITHOUT ANY WARRANTY; without even the implied warranty of               #

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #

# GNU General Public License for more details.                                 #

#                                                                              #

# You should have received a copy of the GNU General Public License            #

# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #

#                                                                              #

# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################





find_path(Hercules_INCLUDE_DIR

  NAMES user_functions.hpp

  PREFIX hercules

)



find_library(Hercules_LIBRARY

  NAMES libhercules_user_lib.so

)



include(FindPackageHandleStandardArgs)

find_package_handle_standard_args(

	Hercules

	DEFAULT_MSG

	Hercules_INCLUDE_DIR

	Hercules_LIBRARY

)



if(Hercules_FOUND)

  set(Hercules_LIBRARIES ${Hercules_LIBRARY})

  set(Hercules_INCLUDE_DIRS ${Hercules_INCLUDE_DIR})





  if(NOT TARGET Hercules::Hercules)

	  add_library(Hercules::Hercules UNKNOWN IMPORTED)

	  set_target_properties(Hercules::Hercules PROPERTIES

		IMPORTED_LOCATION "${Hercules_LIBRARY}"

		INTERFACE_INCLUDE_DIRECTORIES "${Hercules_INCLUDE_DIR}"

	  )

	endif()

endif()





mark_as_advanced(

	Hercules_INCLUDE_DIR

	Hercules_LIBRARY

)

lib/cargo.hpp

+4 −1
Changes for lib/cargo.hpp: 4 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -77,6 +77,9 @@ public: 
    void

    path(std::string path);



    [[nodiscard]] dataset::type

    get_type() const;



    template <typename Archive>

    void

    serialize(Archive& ar) {
@@ -86,7 +89,7 @@ public: 


private:

    std::string m_path;

    dataset::type m_type = dataset::type::none;

    dataset::type m_type;

};