Loading include/api/connection.hpp +9 −4 Original line number Diff line number Diff line Loading @@ -26,10 +26,11 @@ SPDX-License-Identifier: GPL-3.0-or-later */ #ifndef GEKKOFS_API_FS_HPP #define GEKKOFS_API_FS_HPP #ifndef GEKKOFS_API_CONNECTION_HPP #define GEKKOFS_API_CONNECTION_HPP #include <memory> #include <filesystem> namespace gkfs::api::fs { Loading @@ -37,7 +38,7 @@ class connection { public: connection(); connection(const std::string& hostfile); explicit connection(const std::string& hostfile); ~connection(); connection(connection&& rhs) noexcept = default; Loading @@ -47,6 +48,10 @@ public: connection& operator=(const connection& rhs) = delete; void put(const void* buffer, size_t size, const std::filesystem::path& gkfs_filename); private: class impl; std::unique_ptr<impl> pimpl_; Loading @@ -54,4 +59,4 @@ private: } // namespace gkfs::api::fs #endif // GEKKOFS_API_FS_HPP #endif // GEKKOFS_API_CONNECTION_HPP No newline at end of file include/api/detail/connection_impl.hpp 0 → 100644 +144 −0 Original line number Diff line number Diff line /* Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany This software was partially supported by the EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). This software was partially supported by the ADA-FS project under the SPPEXA project funded by the DFG. This file is part of GekkoFS. GekkoFS is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. GekkoFS is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with GekkoFS. If not, see <https://www.gnu.org/licenses/>. SPDX-License-Identifier: GPL-3.0-or-later */ #ifndef GEKKOFS_API_DETAIL_CONNECTION_IMPL_HPP #define GEKKOFS_API_DETAIL_CONNECTION_IMPL_HPP #include <memory> #include <vector> #include <hermes.hpp> // forward declarations namespace gkfs::metadata { class Metadata; } namespace gkfs::rpc { class Distributor; } namespace gkfs::api::fs { namespace detail { struct peer_list { using endpoint_type = hermes::endpoint; peer_list(uint64_t localhost_id, std::vector<endpoint_type> endpoints) : localhost_id_(localhost_id), endpoints_(std::move(endpoints)) {} uint64_t localhost_id() const { return localhost_id_; } hermes::endpoint localhost() const { return endpoints_.at(localhost_id_); } std::size_t size() const { return endpoints_.size(); } hermes::endpoint at(std::size_t n) const { return endpoints_.at(n); } uint64_t localhost_id_ = 0; std::vector<hermes::endpoint> endpoints_; }; } // namespace detail class connection::impl { public: explicit impl(const std::string_view& hostfile); impl(const impl& other) = delete; impl(impl&& rhs) noexcept; impl& operator=(const impl& other) = delete; impl& operator=(impl&& rhs) noexcept; ~impl(); void put(const void* buffer, size_t size, const std::filesystem::path& gkfs_filename) const; std::string stat(const std::string& path) const; void create(const std::string& gkfs_filename, mode_t mode) const; ssize_t pwrite(const std::string& gkfs_filename, const void* buffer, size_t size, off_t offset) const; private: void check_parent_directory(const std::string& path) const; std::optional<gkfs::metadata::Metadata> fetch_metadata(const std::string& path, bool follow_links = true) const; int forward_create_rpc(const std::string& gkfs_filename, const mode_t mode) const; std::tuple<int, off64_t> forward_update_metadentry_size_rpc(const std::string& gkfs_filename, size_t size, off_t offset, bool is_append) const; std::tuple<int, ssize_t> forward_write_rpc(const std::string& gkfs_filename, const void* buffer, off_t offset, size_t write_size, size_t updated_metadentry_size, bool is_append) const; private: //////////////////////////////////////////////////////////////////////////// // members // //////////////////////////////////////////////////////////////////////////// std::vector<fs::host> hosts_; rpc::config rpc_config_; std::unique_ptr<hermes::async_engine> rpc_service_; detail::peer_list peers_; std::vector<hermes::endpoint> host_endpoints_; fs::config fs_config_; std::unique_ptr<gkfs::rpc::Distributor> distributor_; }; } // namespace gkfs::api::fs #endif // GEKKOFS_API_DETAIL_CONNECTION_IMPL_HPP No newline at end of file include/api/env.hpp 0 → 100644 +58 −0 Original line number Diff line number Diff line /* Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany This software was partially supported by the EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). This software was partially supported by the ADA-FS project under the SPPEXA project funded by the DFG. This file is part of GekkoFS' POSIX interface. GekkoFS' POSIX interface is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. GekkoFS' POSIX interface is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with GekkoFS' POSIX interface. If not, see <https://www.gnu.org/licenses/>. SPDX-License-Identifier: LGPL-3.0-or-later */ #ifndef GKFS_API_ENV #define GKFS_API_ENV #include <config.hpp> #define ADD_PREFIX(str) API_ENV_PREFIX str /* Environment variables for the GekkoFS client */ namespace gkfs::env { [[maybe_unused]] static constexpr auto LOG = ADD_PREFIX("LOG"); #ifdef GKFS_DEBUG_BUILD [[maybe_unused]] static constexpr auto LOG_DEBUG_VERBOSITY = ADD_PREFIX("LOG_DEBUG_VERBOSITY"); #endif [[maybe_unused]] static constexpr auto LOG_OUTPUT = ADD_PREFIX("LOG_OUTPUT"); [[maybe_unused]] static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); [[maybe_unused]] static constexpr auto CWD = ADD_PREFIX("CWD"); static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); #ifdef GKFS_ENABLE_FORWARDING static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE"); #endif } // namespace gkfs::env #undef ADD_PREFIX #endif // GKFS_API_ENV No newline at end of file include/api/staging.hpp +49 −1 Original line number Diff line number Diff line Loading @@ -33,8 +33,56 @@ namespace gkfs::api::staging { /** * Transfer data into a GekkoFS file. * * This function transfers data from the provided buffer into a target * file in GekkoFS in one operation. If the file does not exist, it will be * created. This function creates an internal connection to the filesystem and * destroys it after the operation completes. * * @param buffer The buffer with the data contents * @param size The amount of data to transfer * @param gkfs_pathname The target filename inside GekkoFS * @param offset The offset in the file where the data will be placed. */ void put(const void* buffer, size_t size, const std::filesystem::path& gkfs_filename, off_t offset); /** * Transfer data into a GekkoFS file. * * This function transfers data from the provided buffer into a target * file in GekkoFS in one operation. If the file does not exist, it will be * created. The function is capable of reusing an existing connection * to the filesystem, which might be beneficial for performance reasons, but * leaves the responsibility of creating and destroying the connection to the * user. * * @param conn An existing connection for GekkoFS * @param buffer The buffer with the data contents * @param size The amount of data to transfer * @param gkfs_pathname The target filename inside GekkoFS */ void put(const gkfs::api::fs::connection& conn, const void* buffer, size_t size, const std::filesystem::path& gkfs_filename); /** * Transfer data into a GekkoFS file. * * This function creates an internal connection to a running GekkoFS * filesystem and transfers data from the provided buffer into a target * file. * * @param buffer The buffer with the data contents * @param size The amount of data to write * @param gkfs_pathname The target filename inside GekkoFS * @param offset */ void put(const std::filesystem::path& pathname, const void* buf, size_t count, put(const void* buffer, size_t size, const std::filesystem::path& gkfs_filename, off_t offset); void Loading include/client/env.hpp +5 −5 Original line number Diff line number Diff line Loading @@ -37,16 +37,16 @@ /* Environment variables for the GekkoFS client */ namespace gkfs::env { static constexpr auto LOG = ADD_PREFIX("LOG"); [[maybe_unused]] [[maybe_unused]] static constexpr auto LOG = ADD_PREFIX("LOG"); #ifdef GKFS_DEBUG_BUILD static constexpr auto LOG_DEBUG_VERBOSITY = ADD_PREFIX("LOG_DEBUG_VERBOSITY"); [[maybe_unused]] [[maybe_unused]] static constexpr auto LOG_DEBUG_VERBOSITY = ADD_PREFIX("LOG_DEBUG_VERBOSITY"); static constexpr auto LOG_SYSCALL_FILTER = ADD_PREFIX("LOG_SYSCALL_FILTER"); #endif static constexpr auto LOG_OUTPUT = ADD_PREFIX("LOG_OUTPUT"); static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); static constexpr auto CWD = ADD_PREFIX("CWD"); [[maybe_unused]] static constexpr auto LOG_OUTPUT = ADD_PREFIX("LOG_OUTPUT"); [[maybe_unused]] static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); [[maybe_unused]] static constexpr auto CWD = ADD_PREFIX("CWD"); static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); #ifdef GKFS_ENABLE_FORWARDING static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE"); Loading Loading
include/api/connection.hpp +9 −4 Original line number Diff line number Diff line Loading @@ -26,10 +26,11 @@ SPDX-License-Identifier: GPL-3.0-or-later */ #ifndef GEKKOFS_API_FS_HPP #define GEKKOFS_API_FS_HPP #ifndef GEKKOFS_API_CONNECTION_HPP #define GEKKOFS_API_CONNECTION_HPP #include <memory> #include <filesystem> namespace gkfs::api::fs { Loading @@ -37,7 +38,7 @@ class connection { public: connection(); connection(const std::string& hostfile); explicit connection(const std::string& hostfile); ~connection(); connection(connection&& rhs) noexcept = default; Loading @@ -47,6 +48,10 @@ public: connection& operator=(const connection& rhs) = delete; void put(const void* buffer, size_t size, const std::filesystem::path& gkfs_filename); private: class impl; std::unique_ptr<impl> pimpl_; Loading @@ -54,4 +59,4 @@ private: } // namespace gkfs::api::fs #endif // GEKKOFS_API_FS_HPP #endif // GEKKOFS_API_CONNECTION_HPP No newline at end of file
include/api/detail/connection_impl.hpp 0 → 100644 +144 −0 Original line number Diff line number Diff line /* Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany This software was partially supported by the EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). This software was partially supported by the ADA-FS project under the SPPEXA project funded by the DFG. This file is part of GekkoFS. GekkoFS is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. GekkoFS is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with GekkoFS. If not, see <https://www.gnu.org/licenses/>. SPDX-License-Identifier: GPL-3.0-or-later */ #ifndef GEKKOFS_API_DETAIL_CONNECTION_IMPL_HPP #define GEKKOFS_API_DETAIL_CONNECTION_IMPL_HPP #include <memory> #include <vector> #include <hermes.hpp> // forward declarations namespace gkfs::metadata { class Metadata; } namespace gkfs::rpc { class Distributor; } namespace gkfs::api::fs { namespace detail { struct peer_list { using endpoint_type = hermes::endpoint; peer_list(uint64_t localhost_id, std::vector<endpoint_type> endpoints) : localhost_id_(localhost_id), endpoints_(std::move(endpoints)) {} uint64_t localhost_id() const { return localhost_id_; } hermes::endpoint localhost() const { return endpoints_.at(localhost_id_); } std::size_t size() const { return endpoints_.size(); } hermes::endpoint at(std::size_t n) const { return endpoints_.at(n); } uint64_t localhost_id_ = 0; std::vector<hermes::endpoint> endpoints_; }; } // namespace detail class connection::impl { public: explicit impl(const std::string_view& hostfile); impl(const impl& other) = delete; impl(impl&& rhs) noexcept; impl& operator=(const impl& other) = delete; impl& operator=(impl&& rhs) noexcept; ~impl(); void put(const void* buffer, size_t size, const std::filesystem::path& gkfs_filename) const; std::string stat(const std::string& path) const; void create(const std::string& gkfs_filename, mode_t mode) const; ssize_t pwrite(const std::string& gkfs_filename, const void* buffer, size_t size, off_t offset) const; private: void check_parent_directory(const std::string& path) const; std::optional<gkfs::metadata::Metadata> fetch_metadata(const std::string& path, bool follow_links = true) const; int forward_create_rpc(const std::string& gkfs_filename, const mode_t mode) const; std::tuple<int, off64_t> forward_update_metadentry_size_rpc(const std::string& gkfs_filename, size_t size, off_t offset, bool is_append) const; std::tuple<int, ssize_t> forward_write_rpc(const std::string& gkfs_filename, const void* buffer, off_t offset, size_t write_size, size_t updated_metadentry_size, bool is_append) const; private: //////////////////////////////////////////////////////////////////////////// // members // //////////////////////////////////////////////////////////////////////////// std::vector<fs::host> hosts_; rpc::config rpc_config_; std::unique_ptr<hermes::async_engine> rpc_service_; detail::peer_list peers_; std::vector<hermes::endpoint> host_endpoints_; fs::config fs_config_; std::unique_ptr<gkfs::rpc::Distributor> distributor_; }; } // namespace gkfs::api::fs #endif // GEKKOFS_API_DETAIL_CONNECTION_IMPL_HPP No newline at end of file
include/api/env.hpp 0 → 100644 +58 −0 Original line number Diff line number Diff line /* Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany This software was partially supported by the EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). This software was partially supported by the ADA-FS project under the SPPEXA project funded by the DFG. This file is part of GekkoFS' POSIX interface. GekkoFS' POSIX interface is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. GekkoFS' POSIX interface is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with GekkoFS' POSIX interface. If not, see <https://www.gnu.org/licenses/>. SPDX-License-Identifier: LGPL-3.0-or-later */ #ifndef GKFS_API_ENV #define GKFS_API_ENV #include <config.hpp> #define ADD_PREFIX(str) API_ENV_PREFIX str /* Environment variables for the GekkoFS client */ namespace gkfs::env { [[maybe_unused]] static constexpr auto LOG = ADD_PREFIX("LOG"); #ifdef GKFS_DEBUG_BUILD [[maybe_unused]] static constexpr auto LOG_DEBUG_VERBOSITY = ADD_PREFIX("LOG_DEBUG_VERBOSITY"); #endif [[maybe_unused]] static constexpr auto LOG_OUTPUT = ADD_PREFIX("LOG_OUTPUT"); [[maybe_unused]] static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); [[maybe_unused]] static constexpr auto CWD = ADD_PREFIX("CWD"); static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); #ifdef GKFS_ENABLE_FORWARDING static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE"); #endif } // namespace gkfs::env #undef ADD_PREFIX #endif // GKFS_API_ENV No newline at end of file
include/api/staging.hpp +49 −1 Original line number Diff line number Diff line Loading @@ -33,8 +33,56 @@ namespace gkfs::api::staging { /** * Transfer data into a GekkoFS file. * * This function transfers data from the provided buffer into a target * file in GekkoFS in one operation. If the file does not exist, it will be * created. This function creates an internal connection to the filesystem and * destroys it after the operation completes. * * @param buffer The buffer with the data contents * @param size The amount of data to transfer * @param gkfs_pathname The target filename inside GekkoFS * @param offset The offset in the file where the data will be placed. */ void put(const void* buffer, size_t size, const std::filesystem::path& gkfs_filename, off_t offset); /** * Transfer data into a GekkoFS file. * * This function transfers data from the provided buffer into a target * file in GekkoFS in one operation. If the file does not exist, it will be * created. The function is capable of reusing an existing connection * to the filesystem, which might be beneficial for performance reasons, but * leaves the responsibility of creating and destroying the connection to the * user. * * @param conn An existing connection for GekkoFS * @param buffer The buffer with the data contents * @param size The amount of data to transfer * @param gkfs_pathname The target filename inside GekkoFS */ void put(const gkfs::api::fs::connection& conn, const void* buffer, size_t size, const std::filesystem::path& gkfs_filename); /** * Transfer data into a GekkoFS file. * * This function creates an internal connection to a running GekkoFS * filesystem and transfers data from the provided buffer into a target * file. * * @param buffer The buffer with the data contents * @param size The amount of data to write * @param gkfs_pathname The target filename inside GekkoFS * @param offset */ void put(const std::filesystem::path& pathname, const void* buf, size_t count, put(const void* buffer, size_t size, const std::filesystem::path& gkfs_filename, off_t offset); void Loading
include/client/env.hpp +5 −5 Original line number Diff line number Diff line Loading @@ -37,16 +37,16 @@ /* Environment variables for the GekkoFS client */ namespace gkfs::env { static constexpr auto LOG = ADD_PREFIX("LOG"); [[maybe_unused]] [[maybe_unused]] static constexpr auto LOG = ADD_PREFIX("LOG"); #ifdef GKFS_DEBUG_BUILD static constexpr auto LOG_DEBUG_VERBOSITY = ADD_PREFIX("LOG_DEBUG_VERBOSITY"); [[maybe_unused]] [[maybe_unused]] static constexpr auto LOG_DEBUG_VERBOSITY = ADD_PREFIX("LOG_DEBUG_VERBOSITY"); static constexpr auto LOG_SYSCALL_FILTER = ADD_PREFIX("LOG_SYSCALL_FILTER"); #endif static constexpr auto LOG_OUTPUT = ADD_PREFIX("LOG_OUTPUT"); static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); static constexpr auto CWD = ADD_PREFIX("CWD"); [[maybe_unused]] static constexpr auto LOG_OUTPUT = ADD_PREFIX("LOG_OUTPUT"); [[maybe_unused]] static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); [[maybe_unused]] static constexpr auto CWD = ADD_PREFIX("CWD"); static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); #ifdef GKFS_ENABLE_FORWARDING static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE"); Loading
