.gitmodules

 [submodule "external/hermes"]
 	path = external/hermes
-	url = https://github.com/bsc-ssrg/hermes.git
+	url = https://github.com/marcvef/hermes.git
 [submodule "tests/scripts/bats"]
 	path = tests/scripts/bats
 	url = https://github.com/bats-core/bats-core.git

CMakeLists.txt

@@ -252,7 +252,7 @@ include_from_source(cli11
 ## Check configured variables/options and act accordingly
 ################################################################################
 if(GKFS_BUILD_DOCUMENTATION)
-    add_subdirectory(docs)
+  add_subdirectory(docs)
 endif()
 
 if (GKFS_SYMLINK_SUPPORT)
@@ -309,6 +309,7 @@ include_directories(
 
 add_subdirectory(src)
 add_subdirectory(include)
+#add_subdirectory(marc)
 
 ### Mark any CMake variables imported from {fmt} and spdlog as advanced, so
 ### that they don't appear in cmake-gui or ccmake. Similarly for FETCHCONTENT

hermes @ 7a234d85

-Subproject commit d8755039c82501323cdc9a4f5a2b922b9b3bcac7
+Subproject commit 7a234d854ae24bb33b46319f6735e44ab5dc33c7

include/CMakeLists.txt

@@ -32,8 +32,11 @@ add_subdirectory(common)
 add_subdirectory(daemon)
 # Client library
 add_subdirectory(client)
+# Proxy
+add_subdirectory(proxy)
 
 target_sources(gkfs_daemon PUBLIC config.hpp version.hpp.in)
+target_sources(gkfs_proxy PUBLIC config.hpp version.hpp.in)
 
 if(GKFS_ENABLE_FORWARDING)
   target_sources(gkfwd_daemon PUBLIC config.hpp version.hpp.in)

include/client/env.hpp

@@ -52,6 +52,7 @@ static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE");
 #ifdef GKFS_ENABLE_FORWARDING
 static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE");
 #endif
+static constexpr auto PROXY_PID_FILE = ADD_PREFIX("PROXY_PID_FILE");
 
 } // namespace gkfs::env
 

include/client/hooks.hpp

@@ -199,6 +199,9 @@ hook_fsync(unsigned int fd);
 int
 hook_getxattr(const char* path, const char* name, void* value, size_t size);
 
+int
+hook_lgetxattr(const char* path, const char* name, void* value, size_t size);
+
 } // namespace gkfs::hook
 
 #endif

include/client/preload_context.hpp

@@ -98,6 +98,11 @@ private:
     std::string rpc_protocol_;
     bool auto_sm_{false};
 
+    // proxy stuff
+    bool use_proxy_{false};
+    std::string proxy_address_str_;
+    hermes::endpoint proxy_host_;
+
     bool interception_enabled_;
 
     std::bitset<GKFS_MAX_INTERNAL_FDS> internal_fds_;
@@ -178,6 +183,27 @@ public:
     relativize_path(const char* raw_path, std::string& relative_path,
                     bool resolve_last_link = true) const;
 
+    bool
+    use_proxy() const;
+
+    void
+    use_proxy(bool use_proxy);
+
+    const std::string&
+    proxy_address_str() const;
+
+    void
+    proxy_address_str(const std::string& proxy_address_str);
+
+    const hermes::endpoint&
+    proxy_host() const;
+
+    void
+    proxy_host(const hermes::endpoint& proxy_host);
+
+    void
+    clear_proxy_host();
+
     const std::shared_ptr<gkfs::filemap::OpenFileMap>&
     file_map() const;
 

include/client/preload_util.hpp

@@ -61,6 +61,7 @@ class async_engine;
 }
 
 extern std::unique_ptr<hermes::async_engine> ld_network_service;
+extern std::unique_ptr<hermes::async_engine> ld_proxy_service;
 
 // function definitions
 namespace gkfs::utils {
@@ -89,6 +90,13 @@ read_hosts_file();
 void
 connect_to_hosts(const std::vector<std::pair<std::string, std::string>>& hosts);
 
+void
+check_for_proxy();
+
+void
+lookup_proxy_addr();
+
 } // namespace gkfs::utils
 
+
 #endif // GEKKOFS_PRELOAD_UTIL_HPP

include/client/rpc/forward_data.hpp

@@ -30,13 +30,9 @@
 #ifndef GEKKOFS_CLIENT_FORWARD_DATA_HPP
 #define GEKKOFS_CLIENT_FORWARD_DATA_HPP
 
-namespace gkfs::rpc {
+#include <common/common_defs.hpp>
 
-struct ChunkStat {
-    unsigned long chunk_size;
-    unsigned long chunk_total;
-    unsigned long chunk_free;
-};
+namespace gkfs::rpc {
 
 // TODO once we have LEAF, remove all the error code returns and throw them as
 // an exception.

include/client/rpc/forward_data_proxy.hpp

+/*
+  Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain
+  Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany
+
+  This software was partially supported by the
+  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+  This software was partially supported by the
+  ADA-FS project under the SPPEXA project funded by the DFG.
+
+  SPDX-License-Identifier: MIT
+*/
+
+#ifndef GEKKOFS_FORWARD_DATA_PROXY_HPP
+#define GEKKOFS_FORWARD_DATA_PROXY_HPP
+
+#include <common/common_defs.hpp>
+
+namespace gkfs::rpc {
+
+std::pair<int, ssize_t>
+forward_write_proxy(const std::string& path, const void* buf, off64_t offset,
+                    size_t write_size);
+
+std::pair<int, ssize_t>
+forward_read_proxy(const std::string& path, void* buf, off64_t offset,
+                   size_t read_size);
+
+std::pair<int, ChunkStat>
+forward_get_chunk_stat_proxy();
+
+} // namespace gkfs::rpc
+
+#endif // GEKKOFS_FORWARD_DATA_PROXY_HPP

include/client/rpc/forward_metadata_proxy.hpp

+/*
+  Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain
+  Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany
+
+  This software was partially supported by the
+  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+  This software was partially supported by the
+  ADA-FS project under the SPPEXA project funded by the DFG.
+
+  SPDX-License-Identifier: MIT
+*/
+
+#ifndef GEKKOFS_FORWARD_METADATA_PROXY_HPP
+#define GEKKOFS_FORWARD_METADATA_PROXY_HPP
+
+namespace gkfs::rpc {
+
+int
+forward_create_proxy(const std::string& path, const mode_t mode);
+
+int
+forward_stat_proxy(const std::string& path, std::string& attr);
+
+int
+forward_remove_proxy(const std::string& path);
+
+std::pair<int, off64_t>
+forward_update_metadentry_size_proxy(const std::string& path, const size_t size,
+                                     const off64_t offset,
+                                     const bool append_flag);
+
+std::pair<int, std::vector<std::tuple<const std::string, bool, size_t, time_t>>>
+forward_get_dirents_single_proxy(const std::string& path, int server);
+
+} // namespace gkfs::rpc
+
+#endif // GEKKOFS_FORWARD_METADATA_PROXY_HPP

include/common/CMakeLists.txt

@@ -31,6 +31,12 @@ target_sources(
                      rpc/rpc_util.hpp
 )
 
+target_sources(gkfs_proxy
+    PUBLIC
+    common_defs.hpp
+    rpc/rpc_types.hpp
+    rpc/rpc_util.hpp)
+
 if(GKFS_ENABLE_FORWARDING)
   target_sources(
     gkfwd_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp

include/common/common_defs.hpp

@@ -34,6 +34,12 @@
 namespace gkfs::rpc {
 
 using chnk_id_t = unsigned long;
+struct ChunkStat {
+    unsigned long chunk_size;
+    unsigned long chunk_total;
+    unsigned long chunk_free;
+};
+
 
 namespace tag {
 
@@ -55,6 +61,16 @@ constexpr auto write = "rpc_srv_write_data";
 constexpr auto read = "rpc_srv_read_data";
 constexpr auto truncate = "rpc_srv_trunc_data";
 constexpr auto get_chunk_stat = "rpc_srv_chunk_stat";
+// IPC communication between client and proxy
+constexpr auto proxy_create = "proxy_rpc_srv_create";
+constexpr auto proxy_stat = "proxy_rpc_srv_stat";
+constexpr auto proxy_remove = "proxy_rpc_srv_remove";
+constexpr auto proxy_update_size = "proxy_rpc_srv_update_metadentry_size";
+constexpr auto proxy_write = "proxy_rpc_srv_write_data";
+constexpr auto proxy_read = "proxy_rpc_srv_read_data";
+constexpr auto proxy_chunk_stat = "proxy_rpc_srv_chunk_stat";
+constexpr auto proxy_get_dirents_extended =
+        "proxy_rpc_srv_get_dirents_extended";
 } // namespace tag
 
 namespace protocol {

include/common/rpc/rpc_types.hpp

@@ -126,4 +126,22 @@ MERCURY_GEN_PROC(
         ((hg_int32_t) (err))((hg_uint64_t) (chunk_size))(
                 (hg_uint64_t) (chunk_total))((hg_uint64_t) (chunk_free)))
 
+// proxy
+MERCURY_GEN_PROC(rpc_proxy_write_data_in_t,
+                 ((hg_const_string_t) (path))(
+                         (int64_t) (offset)) // file offset, NOT chunk offset
+                 ((hg_uint64_t) (write_size))((hg_bulk_t) (bulk_handle)))
+
+MERCURY_GEN_PROC(rpc_proxy_read_data_in_t,
+                 ((hg_const_string_t) (path))(
+                         (int64_t) (offset)) // file offset, NOT chunk offset
+                 ((hg_uint64_t) (read_size))((hg_bulk_t) (bulk_handle)))
+
+MERCURY_GEN_PROC(rpc_proxy_test_in_t, ((hg_const_string_t) (path)))
+
+MERCURY_GEN_PROC(rpc_proxy_get_dirents_in_t,
+                 ((hg_const_string_t) (path))((int32_t) (server))(
+                         (hg_bulk_t) (bulk_handle)))
+
+
 #endif // LFS_RPC_TYPES_HPP

include/config.hpp

@@ -35,9 +35,13 @@
 #define CLIENT_ENV_PREFIX "LIBGKFS_"
 #define DAEMON_ENV_PREFIX "GKFS_DAEMON_"
 #define COMMON_ENV_PREFIX "GKFS_"
+#define PROXY_ENV_PREFIX  "GKFS_PROXY_"
 
 namespace gkfs::config {
 
+// writes to dev null instead of chunk space, read is reading /dev/zero
+constexpr bool limbo_mode = false;
+
 constexpr auto hostfile_path = "./gkfs_hosts.txt";
 constexpr auto forwarding_file_path = "./gkfs_forwarding.map";
 
@@ -47,14 +51,28 @@ namespace io {
  * If buffer is not zeroed, sparse regions contain invalid data.
  */
 constexpr auto zero_buffer_before_read = false;
+/*
+ * When the daemon handler serves a read request, it starts tasklets (for each
+ * chunk) from the io pool to read all chunks of that read request in parallel.
+ * Then another thread is waiting for the first tasklet to finish before
+ * initiating the bulk transfer back to the client for this chunk.
+ * This will continue in sequence, allowing gaps between bulk transfers while
+ * waiting. Although this is CPU efficient, it does not provide the highest I/O.
+ * if spin_lock_read is enabled it will try all tasklets if they are finished
+ * regardless of their order minimizing the gap between bulk transfers.
+ * Due to spinning in a loop this increases CPU utilization
+ */
+constexpr auto spin_lock_read = true;
 } // namespace io
 
 namespace log {
 constexpr auto client_log_path = "/tmp/gkfs_client.log";
 constexpr auto daemon_log_path = "/tmp/gkfs_daemon.log";
+constexpr auto proxy_log_path = "/tmp/gkfs_proxy.log";
 
 constexpr auto client_log_level = "info,errors,critical,hermes";
 constexpr auto daemon_log_level = 4; // info
+constexpr auto proxy_log_level = 4;  // info
 } // namespace log
 
 namespace metadata {
@@ -90,6 +108,20 @@ namespace data {
 constexpr auto chunk_dir = "chunks";
 } // namespace data
 
+namespace proxy {
+constexpr auto pid_path = "/tmp/gkfs_proxy.pid";
+constexpr auto fwd_create = true;
+constexpr auto fwd_stat = true;
+constexpr auto fwd_remove = true;
+constexpr auto fwd_update_size = true;
+constexpr auto fwd_io = true;
+constexpr auto fwd_chunk_stat = true;
+constexpr auto fwd_get_dirents_single = true;
+// Only use proxy for io if write/read size is higher than set value
+constexpr auto fwd_io_count_threshold = 0;
+
+} // namespace proxy
+
 namespace rpc {
 constexpr auto chunksize = 524288; // in bytes (e.g., 524288 == 512KB)
 // size of preallocated buffer to hold directory entries in rpc call
@@ -102,6 +134,8 @@ constexpr auto dirents_buff_size = (8 * 1024 * 1024); // 8 mega
 constexpr auto daemon_io_xstreams = 8;
 // Number of threads used for RPC handlers at the daemon
 constexpr auto daemon_handler_xstreams = 4;
+// Number of threads used for RPC handlers at the proxy
+constexpr auto proxy_handler_xstreams = 3;
 } // namespace rpc
 
 namespace rocksdb {

include/daemon/classes/fs_data.hpp

@@ -69,7 +69,10 @@ private:
 
     // RPC management
     std::string rpc_protocol_{};
+    std::string proxy_rpc_protocol_{};
     std::string bind_addr_{};
+    std::string bind_proxy_addr_{}; // optional when used with running proxy.
+                                    // Remains empty if unused
     std::string hosts_file_{};
     bool use_auto_sm_;
 
@@ -171,12 +174,23 @@ public:
     void
     rpc_protocol(const std::string& rpc_protocol);
 
+    const std::string&
+    proxy_rpc_protocol() const;
+    void
+    proxy_rpc_protocol(const std::string& proxy_rpc_protocol);
+
     const std::string&
     bind_addr() const;
 
     void
     bind_addr(const std::string& addr);
 
+    const std::string&
+    bind_proxy_addr() const;
+
+    void
+    bind_proxy_addr(const std::string& proxy_addr);
+
     const std::string&
     hosts_file() const;
 

include/daemon/classes/rpc_data.hpp

@@ -49,11 +49,13 @@ private:
     // Margo IDs. They can also be used to retrieve the Mercury classes and
     // contexts that were created at init time
     margo_instance_id server_rpc_mid_;
+    margo_instance_id proxy_server_rpc_mid_;
 
     // Argobots I/O pools and execution streams
     ABT_pool io_pool_;
     std::vector<ABT_xstream> io_streams_;
     std::string self_addr_str_;
+    std::string self_proxy_addr_str_;
     // Distributor
     std::shared_ptr<gkfs::rpc::Distributor> distributor_;
 
@@ -77,6 +79,12 @@ public:
     void
     server_rpc_mid(margo_instance* server_rpc_mid);
 
+    margo_instance*
+    proxy_server_rpc_mid();
+
+    void
+    proxy_server_rpc_mid(margo_instance* proxy_server_rpc_mid);
+
     ABT_pool
     io_pool() const;
 
@@ -93,7 +101,13 @@ public:
     self_addr_str() const;
 
     void
-    self_addr_str(const std::string& addr_stra);
+    self_addr_str(const std::string& addr_str);
+
+    const std::string&
+    self_proxy_addr_str() const;
+
+    void
+    self_proxy_addr_str(const std::string& proxy_addr_str);
 
     const std::shared_ptr<gkfs::rpc::Distributor>&
     distributor() const;

include/daemon/handler/rpc_defs.hpp

@@ -76,4 +76,9 @@ DECLARE_MARGO_RPC_HANDLER(rpc_srv_truncate)
 
 DECLARE_MARGO_RPC_HANDLER(rpc_srv_get_chunk_stat)
 
+// proxy
+DECLARE_MARGO_RPC_HANDLER(proxy_rpc_srv_read)
+
+DECLARE_MARGO_RPC_HANDLER(proxy_rpc_srv_write)
+
 #endif // GKFS_DAEMON_RPC_DEFS_HPP

include/daemon/ops/data.hpp

@@ -302,7 +302,8 @@ private:
         size_t size;                  //!< size to read from chunk
         off64_t off;                  //!< offset for individual chunk
         ABT_eventual eventual;        //!< Attached eventual
-    };                                //!< Struct for an chunk read operation
+        bool bulk_transfer_done = false;
+    }; //!< Struct for an chunk read operation
 
     std::vector<struct chunk_read_args> task_args_; //!< tasklet input structs
     /**