Draft: Resolve "New feature: data staging API"

Closes #148

Merge request reports

Loading
+247 −0
Changes for CMake/FindFilesystem.cmake: 247 added lines, 0 removed lines.
Original line number Diff line number Diff line
# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
# file Copyright.txt or https://cmake.org/licensing for details.

#[=======================================================================[.rst:

FindFilesystem
##############

This module supports the C++17 standard library's filesystem utilities. Use the
:imp-target:`std::filesystem` imported target to

Options
*******

The ``COMPONENTS`` argument to this module supports the following values:

.. find-component:: Experimental
    :name: fs.Experimental

    Allows the module to find the "experimental" Filesystem TS version of the
    Filesystem library. This is the library that should be used with the
    ``std::experimental::filesystem`` namespace.

.. find-component:: Final
    :name: fs.Final

    Finds the final C++17 standard version of the filesystem library.

If no components are provided, behaves as if the
:find-component:`fs.Final` component was specified.

If both :find-component:`fs.Experimental` and :find-component:`fs.Final` are
provided, first looks for ``Final``, and falls back to ``Experimental`` in case
of failure. If ``Final`` is found, :imp-target:`std::filesystem` and all
:ref:`variables <fs.variables>` will refer to the ``Final`` version.


Imported Targets
****************

.. imp-target:: std::filesystem

    The ``std::filesystem`` imported target is defined when any requested
    version of the C++ filesystem library has been found, whether it is
    *Experimental* or *Final*.

    If no version of the filesystem library is available, this target will not
    be defined.

    .. note::
        This target has ``cxx_std_17`` as an ``INTERFACE``
        :ref:`compile language standard feature <req-lang-standards>`. Linking
        to this target will automatically enable C++17 if no later standard
        version is already required on the linking target.


.. _fs.variables:

Variables
*********

.. variable:: CXX_FILESYSTEM_IS_EXPERIMENTAL

    Set to ``TRUE`` when the :find-component:`fs.Experimental` version of C++
    filesystem library was found, otherwise ``FALSE``.

.. variable:: CXX_FILESYSTEM_HAVE_FS

    Set to ``TRUE`` when a filesystem header was found.

.. variable:: CXX_FILESYSTEM_HEADER

    Set to either ``filesystem`` or ``experimental/filesystem`` depending on
    whether :find-component:`fs.Final` or :find-component:`fs.Experimental` was
    found.

.. variable:: CXX_FILESYSTEM_NAMESPACE

    Set to either ``std::filesystem`` or ``std::experimental::filesystem``
    depending on whether :find-component:`fs.Final` or
    :find-component:`fs.Experimental` was found.


Examples
********

Using `find_package(Filesystem)` with no component arguments:

.. code-block:: cmake

    find_package(Filesystem REQUIRED)

    add_executable(my-program main.cpp)
    target_link_libraries(my-program PRIVATE std::filesystem)


#]=======================================================================]


if(TARGET std::filesystem)
    # This module has already been processed. Don't do it again.
    return()
endif()

cmake_minimum_required(VERSION 3.10)

include(CMakePushCheckState)
include(CheckIncludeFileCXX)

# If we're not cross-compiling, try to run test executables.
# Otherwise, assume that compile + link is a sufficient check.
if(CMAKE_CROSSCOMPILING)
    include(CheckCXXSourceCompiles)
    macro(_cmcm_check_cxx_source code var)
        check_cxx_source_compiles("${code}" ${var})
    endmacro()
else()
    include(CheckCXXSourceRuns)
    macro(_cmcm_check_cxx_source code var)
        check_cxx_source_runs("${code}" ${var})
    endmacro()
endif()

cmake_push_check_state()

set(CMAKE_REQUIRED_QUIET ${Filesystem_FIND_QUIETLY})

# All of our tests required C++17 or later
set(CMAKE_CXX_STANDARD 17)

# Normalize and check the component list we were given
set(want_components ${Filesystem_FIND_COMPONENTS})
if(Filesystem_FIND_COMPONENTS STREQUAL "")
    set(want_components Final)
endif()

# Warn on any unrecognized components
set(extra_components ${want_components})
list(REMOVE_ITEM extra_components Final Experimental)
foreach(component IN LISTS extra_components)
    message(WARNING "Extraneous find_package component for Filesystem: ${component}")
endforeach()

# Detect which of Experimental and Final we should look for
set(find_experimental TRUE)
set(find_final TRUE)
if(NOT "Final" IN_LIST want_components)
    set(find_final FALSE)
endif()
if(NOT "Experimental" IN_LIST want_components)
    set(find_experimental FALSE)
endif()

if(find_final)
    check_include_file_cxx("filesystem" _CXX_FILESYSTEM_HAVE_HEADER)
    mark_as_advanced(_CXX_FILESYSTEM_HAVE_HEADER)
    if(_CXX_FILESYSTEM_HAVE_HEADER)
        # We found the non-experimental header. Don't bother looking for the
        # experimental one.
        set(find_experimental FALSE)
    endif()
else()
    set(_CXX_FILESYSTEM_HAVE_HEADER FALSE)
endif()

if(find_experimental)
    check_include_file_cxx("experimental/filesystem" _CXX_FILESYSTEM_HAVE_EXPERIMENTAL_HEADER)
    mark_as_advanced(_CXX_FILESYSTEM_HAVE_EXPERIMENTAL_HEADER)
else()
    set(_CXX_FILESYSTEM_HAVE_EXPERIMENTAL_HEADER FALSE)
endif()

if(_CXX_FILESYSTEM_HAVE_HEADER)
    set(_have_fs TRUE)
    set(_fs_header filesystem)
    set(_fs_namespace std::filesystem)
    set(_is_experimental FALSE)
elseif(_CXX_FILESYSTEM_HAVE_EXPERIMENTAL_HEADER)
    set(_have_fs TRUE)
    set(_fs_header experimental/filesystem)
    set(_fs_namespace std::experimental::filesystem)
    set(_is_experimental TRUE)
else()
    set(_have_fs FALSE)
endif()

set(CXX_FILESYSTEM_HAVE_FS ${_have_fs} CACHE BOOL "TRUE if we have the C++ filesystem headers")
set(CXX_FILESYSTEM_HEADER ${_fs_header} CACHE STRING "The header that should be included to obtain the filesystem APIs")
set(CXX_FILESYSTEM_NAMESPACE ${_fs_namespace} CACHE STRING "The C++ namespace that contains the filesystem APIs")
set(CXX_FILESYSTEM_IS_EXPERIMENTAL ${_is_experimental} CACHE BOOL "TRUE if the C++ filesystem library is the experimental version")

set(_found FALSE)

if(CXX_FILESYSTEM_HAVE_FS)
    # We have some filesystem library available. Do link checks
    string(CONFIGURE [[
        #include <cstdlib>
        #include <@CXX_FILESYSTEM_HEADER@>

        int main() {
            auto cwd = @CXX_FILESYSTEM_NAMESPACE@::current_path();
            printf("%s", cwd.c_str());
            return EXIT_SUCCESS;
        }
    ]] code @ONLY)

    # Check a simple filesystem program without any linker flags
    _cmcm_check_cxx_source("${code}" CXX_FILESYSTEM_NO_LINK_NEEDED)

    set(can_link ${CXX_FILESYSTEM_NO_LINK_NEEDED})

    if(NOT CXX_FILESYSTEM_NO_LINK_NEEDED)
        set(prev_libraries ${CMAKE_REQUIRED_LIBRARIES})
        # Add the libstdc++ flag
        set(CMAKE_REQUIRED_LIBRARIES ${prev_libraries} -lstdc++fs)
        _cmcm_check_cxx_source("${code}" CXX_FILESYSTEM_STDCPPFS_NEEDED)
        set(can_link ${CXX_FILESYSTEM_STDCPPFS_NEEDED})
        if(NOT CXX_FILESYSTEM_STDCPPFS_NEEDED)
            # Try the libc++ flag
            set(CMAKE_REQUIRED_LIBRARIES ${prev_libraries} -lc++fs)
            _cmcm_check_cxx_source("${code}" CXX_FILESYSTEM_CPPFS_NEEDED)
            set(can_link ${CXX_FILESYSTEM_CPPFS_NEEDED})
        endif()
    endif()

    if(can_link)
        add_library(std::filesystem INTERFACE IMPORTED)
        set_property(TARGET std::filesystem APPEND PROPERTY INTERFACE_COMPILE_FEATURES cxx_std_17)
        set(_found TRUE)

        if(CXX_FILESYSTEM_NO_LINK_NEEDED)
            # Nothing to add...
        elseif(CXX_FILESYSTEM_STDCPPFS_NEEDED)
            set_property(TARGET std::filesystem APPEND PROPERTY INTERFACE_LINK_LIBRARIES -lstdc++fs)
        elseif(CXX_FILESYSTEM_CPPFS_NEEDED)
            set_property(TARGET std::filesystem APPEND PROPERTY INTERFACE_LINK_LIBRARIES -lc++fs)
        endif()
    endif()
endif()

cmake_pop_check_state()

set(Filesystem_FOUND ${_found} CACHE BOOL "TRUE if we can run a program using std::filesystem" FORCE)

if(Filesystem_FIND_REQUIRED AND NOT Filesystem_FOUND)
    message(FATAL_ERROR "Cannot run simple program using std::filesystem")
endif()
Compare 0af45bfa to 97820ed8
Changes for external/hermes: 1 added line, 1 removed line.
Original line number Diff line number Diff line
Subproject commit 0af45bfa667f7ff9c78167ef94d975bffbd879f0
Subproject commit 97820ed81aba9ef7c05891ce80c53a8533b78609
+144 −0
Changes for include/api/detail/connection_impl.hpp: 144 added lines, 0 removed lines.
Original line number Diff line number Diff line
/*
  Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain
  Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany

  This software was partially supported by the
  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).

  This software was partially supported by the
  ADA-FS project under the SPPEXA project funded by the DFG.

  This file is part of GekkoFS.

  GekkoFS is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 3 of the License, or
  (at your option) any later version.

  GekkoFS is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.

  SPDX-License-Identifier: GPL-3.0-or-later
*/

#ifndef GEKKOFS_API_DETAIL_CONNECTION_IMPL_HPP
#define GEKKOFS_API_DETAIL_CONNECTION_IMPL_HPP

#include <memory>
#include <vector>
#include <hermes.hpp>

// forward declarations
namespace gkfs::metadata {
class Metadata;
}

namespace gkfs::rpc {
class Distributor;
}

namespace gkfs::api::fs {

namespace detail {

struct peer_list {

    using endpoint_type = hermes::endpoint;

    peer_list(uint64_t localhost_id, std::vector<endpoint_type> endpoints)
        : localhost_id_(localhost_id), endpoints_(std::move(endpoints)) {}

    uint64_t
    localhost_id() const {
        return localhost_id_;
    }

    hermes::endpoint
    localhost() const {
        return endpoints_.at(localhost_id_);
    }

    std::size_t
    size() const {
        return endpoints_.size();
    }

    hermes::endpoint
    at(std::size_t n) const {
        return endpoints_.at(n);
    }

    uint64_t localhost_id_ = 0;
    std::vector<hermes::endpoint> endpoints_;
};

} // namespace detail

class connection::impl {

public:
    explicit impl(const std::string_view& hostfile);
    impl(const impl& other) = delete;
    impl(impl&& rhs) noexcept;
    impl&
    operator=(const impl& other) = delete;
    impl&
    operator=(impl&& rhs) noexcept;
    ~impl();

    std::size_t
    put(const void* buffer, size_t size,
        const std::filesystem::path& gkfs_filename, off_t start_offset) const;

    std::string
    stat(const std::string& path) const;

    void
    create(const std::string& gkfs_filename, mode_t mode) const;

    ssize_t
    pwrite(const std::string& gkfs_filename, const void* buffer, size_t size,
           off_t offset) const;

private:
    void
    check_parent_directory(const std::string& path) const;

    std::optional<gkfs::metadata::Metadata>
    fetch_metadata(const std::string& path, bool follow_links = true) const;

    int
    forward_create_rpc(const std::string& gkfs_filename,
                       const mode_t mode) const;

    std::tuple<int, off64_t>
    forward_update_metadentry_size_rpc(const std::string& gkfs_filename,
                                       size_t size, off_t offset,
                                       bool is_append) const;

    std::tuple<int, ssize_t>
    forward_write_rpc(const std::string& gkfs_filename, const void* buffer,
                      off_t offset, size_t write_size,
                      size_t updated_metadentry_size, bool is_append) const;

private:
    ////////////////////////////////////////////////////////////////////////////
    // members //
    ////////////////////////////////////////////////////////////////////////////
    std::vector<fs::host> hosts_;
    rpc::config rpc_config_;
    std::unique_ptr<hermes::async_engine> rpc_service_;
    detail::peer_list peers_;
    std::vector<hermes::endpoint> host_endpoints_;
    fs::config fs_config_;
    std::unique_ptr<gkfs::rpc::Distributor> distributor_;
};

} // namespace gkfs::api::fs

#endif // GEKKOFS_API_DETAIL_CONNECTION_IMPL_HPP
 No newline at end of file
+62 −0
Changes for include/api/connection.hpp: 62 added lines, 0 removed lines.
Original line number Diff line number Diff line
/*
  Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain
  Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany

  This software was partially supported by the
  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).

  This software was partially supported by the
  ADA-FS project under the SPPEXA project funded by the DFG.

  This file is part of GekkoFS.

  GekkoFS is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 3 of the License, or
  (at your option) any later version.

  GekkoFS is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.

  SPDX-License-Identifier: GPL-3.0-or-later
*/

#ifndef GEKKOFS_API_CONNECTION_HPP
#define GEKKOFS_API_CONNECTION_HPP

#include <memory>
#include <filesystem>

namespace gkfs::api::fs {

class connection {

public:
    connection();
    explicit connection(const std::string& hostfile);
    ~connection();

    connection(connection&& rhs) noexcept = default;
    connection&
    operator=(connection&& rhs) noexcept = default;
    connection(const connection& other) = delete;
    connection&
    operator=(const connection& rhs) = delete;

    std::size_t
    put(const void* buffer, size_t size,
        const std::filesystem::path& gkfs_filename, off_t start_offset) const;

private:
    class impl;
    std::unique_ptr<impl> pimpl_;
};

} // namespace gkfs::api::fs

#endif // GEKKOFS_API_CONNECTION_HPP
 No newline at end of file

include/api/env.hpp

0 → 100644
+60 −0
Changes for include/api/env.hpp: 60 added lines, 0 removed lines.
Original line number Diff line number Diff line
/*
  Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain
  Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany

  This software was partially supported by the
  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).

  This software was partially supported by the
  ADA-FS project under the SPPEXA project funded by the DFG.

  This file is part of GekkoFS' POSIX interface.

  GekkoFS' POSIX interface is free software: you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public License as
  published by the Free Software Foundation, either version 3 of the License,
  or (at your option) any later version.

  GekkoFS' POSIX interface is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public License
  along with GekkoFS' POSIX interface.  If not, see
  <https://www.gnu.org/licenses/>.

  SPDX-License-Identifier: LGPL-3.0-or-later
*/

#ifndef GKFS_API_ENV
#define GKFS_API_ENV

#include <config.hpp>

#define ADD_PREFIX(str) API_ENV_PREFIX str

/* Environment variables for the GekkoFS client */
namespace gkfs::env {

[[maybe_unused]] static constexpr auto LOG = ADD_PREFIX("LOG");

#ifdef GKFS_DEBUG_BUILD
[[maybe_unused]] static constexpr auto LOG_DEBUG_VERBOSITY =
        ADD_PREFIX("LOG_DEBUG_VERBOSITY");
#endif

[[maybe_unused]] static constexpr auto LOG_OUTPUT = ADD_PREFIX("LOG_OUTPUT");
[[maybe_unused]] static constexpr auto LOG_OUTPUT_TRUNC =
        ADD_PREFIX("LOG_OUTPUT_TRUNC");
[[maybe_unused]] static constexpr auto CWD = ADD_PREFIX("CWD");
static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE");
#ifdef GKFS_ENABLE_FORWARDING
static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE");
#endif

} // namespace gkfs::env

#undef ADD_PREFIX

#endif // GKFS_API_ENV
 No newline at end of file
Loading
Loading