.gitmodules

 [submodule "external/hermes"]
 	path = external/hermes
-	url = https://github.com/gekkofs/hermes.git
+	url = https://github.com/marcvef/hermes.git
 [submodule "tests/scripts/bats"]
 	path = tests/scripts/bats
 	url = https://github.com/bats-core/bats-core.git

CMakeLists.txt

@@ -306,6 +306,7 @@ include_directories(
 
 add_subdirectory(src)
 add_subdirectory(include)
+#add_subdirectory(marc)
 
 ### Mark any CMake variables imported from {fmt} and spdlog as advanced, so
 ### that they don't appear in cmake-gui or ccmake. Similarly for FETCHCONTENT

fmt @ e57ca2e3

-Subproject commit f94b7364b9409f05207c3af3fa4666730e11a854
+Subproject commit e57ca2e3685b160617d3d95fcd9e789c4e06ca88

hermes @ d2f9c48d

-Subproject commit d8755039c82501323cdc9a4f5a2b922b9b3bcac7
+Subproject commit d2f9c48d766bd985a2dd1740a70971fd9a7e0203

include/CMakeLists.txt

@@ -32,5 +32,8 @@ add_subdirectory(common)
 add_subdirectory(daemon)
 # Client library
 add_subdirectory(client)
+# Proxy
+add_subdirectory(proxy)
 
 target_sources(gkfs_daemon PUBLIC config.hpp version.hpp.in)
+target_sources(gkfs_proxy PUBLIC config.hpp version.hpp.in)

include/client/env.hpp

@@ -52,6 +52,8 @@ static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE");
 static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE");
 
 static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL");
+static constexpr auto PROXY_PID_FILE = ADD_PREFIX("PROXY_PID_FILE");
+
 } // namespace gkfs::env
 
 #undef ADD_PREFIX

include/client/hooks.hpp

@@ -205,6 +205,9 @@ hook_fsync(unsigned int fd);
 int
 hook_getxattr(const char* path, const char* name, void* value, size_t size);
 
+int
+hook_lgetxattr(const char* path, const char* name, void* value, size_t size);
+
 } // namespace gkfs::hook
 
 #endif

include/client/preload_context.hpp

@@ -98,6 +98,11 @@ private:
     std::string rpc_protocol_;
     bool auto_sm_{false};
 
+    // proxy stuff
+    bool use_proxy_{false};
+    std::string proxy_address_str_;
+    hermes::endpoint proxy_host_;
+
     bool interception_enabled_;
 
     std::bitset<GKFS_MAX_INTERNAL_FDS> internal_fds_;
@@ -179,6 +184,27 @@ public:
     relativize_path(const char* raw_path, std::string& relative_path,
                     bool resolve_last_link = true) const;
 
+    bool
+    use_proxy() const;
+
+    void
+    use_proxy(bool use_proxy);
+
+    const std::string&
+    proxy_address_str() const;
+
+    void
+    proxy_address_str(const std::string& proxy_address_str);
+
+    const hermes::endpoint&
+    proxy_host() const;
+
+    void
+    proxy_host(const hermes::endpoint& proxy_host);
+
+    void
+    clear_proxy_host();
+
     const std::shared_ptr<gkfs::filemap::OpenFileMap>&
     file_map() const;
 

include/client/preload_util.hpp

@@ -61,6 +61,7 @@ class async_engine;
 }
 
 extern std::unique_ptr<hermes::async_engine> ld_network_service;
+extern std::unique_ptr<hermes::async_engine> ld_proxy_service;
 
 // function definitions
 namespace gkfs::utils {
@@ -89,6 +90,13 @@ read_hosts_file();
 void
 connect_to_hosts(const std::vector<std::pair<std::string, std::string>>& hosts);
 
+void
+check_for_proxy();
+
+void
+lookup_proxy_addr();
+
 } // namespace gkfs::utils
 
+
 #endif // GEKKOFS_PRELOAD_UTIL_HPP

include/client/rpc/forward_data.hpp

@@ -30,17 +30,13 @@
 #ifndef GEKKOFS_CLIENT_FORWARD_DATA_HPP
 #define GEKKOFS_CLIENT_FORWARD_DATA_HPP
 
+#include <common/common_defs.hpp>
+
 #include <string>
 #include <memory>
 #include <set>
 namespace gkfs::rpc {
 
-struct ChunkStat {
-    unsigned long chunk_size;
-    unsigned long chunk_total;
-    unsigned long chunk_free;
-};
-
 // TODO once we have LEAF, remove all the error code returns and throw them as
 // an exception.
 

include/client/rpc/forward_data_proxy.hpp

+/*
+  Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain
+  Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany
+
+  This software was partially supported by the
+  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+  This software was partially supported by the
+  ADA-FS project under the SPPEXA project funded by the DFG.
+
+  SPDX-License-Identifier: MIT
+*/
+
+#ifndef GEKKOFS_FORWARD_DATA_PROXY_HPP
+#define GEKKOFS_FORWARD_DATA_PROXY_HPP
+
+#include <common/common_defs.hpp>
+
+namespace gkfs::rpc {
+
+std::pair<int, ssize_t>
+forward_write_proxy(const std::string& path, const void* buf, off64_t offset,
+                    size_t write_size);
+
+std::pair<int, ssize_t>
+forward_read_proxy(const std::string& path, void* buf, off64_t offset,
+                   size_t read_size);
+
+std::pair<int, ChunkStat>
+forward_get_chunk_stat_proxy();
+
+} // namespace gkfs::rpc
+
+#endif // GEKKOFS_FORWARD_DATA_PROXY_HPP

include/client/rpc/forward_metadata_proxy.hpp

+/*
+  Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain
+  Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany
+
+  This software was partially supported by the
+  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+  This software was partially supported by the
+  ADA-FS project under the SPPEXA project funded by the DFG.
+
+  SPDX-License-Identifier: MIT
+*/
+
+#ifndef GEKKOFS_FORWARD_METADATA_PROXY_HPP
+#define GEKKOFS_FORWARD_METADATA_PROXY_HPP
+
+namespace gkfs::rpc {
+
+int
+forward_create_proxy(const std::string& path, const mode_t mode);
+
+int
+forward_stat_proxy(const std::string& path, std::string& attr);
+
+int
+forward_remove_proxy(const std::string& path);
+
+std::pair<int, off64_t>
+forward_update_metadentry_size_proxy(const std::string& path, const size_t size,
+                                     const off64_t offset,
+                                     const bool append_flag);
+
+std::pair<int, std::vector<std::tuple<const std::string, bool, size_t, time_t>>>
+forward_get_dirents_single_proxy(const std::string& path, int server);
+
+} // namespace gkfs::rpc
+
+#endif // GEKKOFS_FORWARD_METADATA_PROXY_HPP

include/common/CMakeLists.txt

@@ -30,3 +30,9 @@ target_sources(
   gkfs_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp rpc/rpc_types.hpp
                      rpc/rpc_util.hpp
 )
+
+target_sources(gkfs_proxy
+    PUBLIC
+    common_defs.hpp
+    rpc/rpc_types.hpp
+    rpc/rpc_util.hpp)
\ No newline at end of file

include/common/common_defs.hpp

@@ -34,6 +34,12 @@
 namespace gkfs::rpc {
 
 using chnk_id_t = unsigned long;
+struct ChunkStat {
+    unsigned long chunk_size;
+    unsigned long chunk_total;
+    unsigned long chunk_free;
+};
+
 
 namespace tag {
 
@@ -55,6 +61,20 @@ constexpr auto write = "rpc_srv_write_data";
 constexpr auto read = "rpc_srv_read_data";
 constexpr auto truncate = "rpc_srv_trunc_data";
 constexpr auto get_chunk_stat = "rpc_srv_chunk_stat";
+// IPC communication between client and proxy
+constexpr auto client_proxy_create = "proxy_rpc_srv_create";
+constexpr auto client_proxy_stat = "proxy_rpc_srv_stat";
+constexpr auto client_proxy_remove = "proxy_rpc_srv_remove";
+constexpr auto client_proxy_update_size =
+        "proxy_rpc_srv_update_metadentry_size";
+constexpr auto client_proxy_write = "proxy_rpc_srv_write_data";
+constexpr auto client_proxy_read = "proxy_rpc_srv_read_data";
+constexpr auto client_proxy_chunk_stat = "proxy_rpc_srv_chunk_stat";
+constexpr auto client_proxy_get_dirents_extended =
+        "proxy_rpc_srv_get_dirents_extended";
+// Specific RPCs between daemon and proxy
+constexpr auto proxy_daemon_write = "proxy_daemon_rpc_srv_write_data";
+constexpr auto proxy_daemon_read = "proxy_daemon_rpc_srv_read_data";
 } // namespace tag
 
 namespace protocol {

include/common/rpc/rpc_types.hpp

@@ -126,4 +126,40 @@ MERCURY_GEN_PROC(
         ((hg_int32_t) (err))((hg_uint64_t) (chunk_size))(
                 (hg_uint64_t) (chunk_total))((hg_uint64_t) (chunk_free)))
 
+// client <-> proxy
+MERCURY_GEN_PROC(rpc_client_proxy_write_in_t,
+                 ((hg_const_string_t) (path))(
+                         (int64_t) (offset)) // file offset, NOT chunk offset
+                 ((hg_uint64_t) (write_size))((hg_bulk_t) (bulk_handle)))
+
+MERCURY_GEN_PROC(rpc_client_proxy_read_in_t,
+                 ((hg_const_string_t) (path))(
+                         (int64_t) (offset)) // file offset, NOT chunk offset
+                 ((hg_uint64_t) (read_size))((hg_bulk_t) (bulk_handle)))
+
+// proxy <-> daemon
+
+MERCURY_GEN_PROC(
+        rpc_proxy_daemon_write_in_t,
+        ((hg_const_string_t) (path))((int64_t) (offset))(
+                (hg_uint64_t) (host_id))((hg_uint64_t) (host_size))(
+                (hg_uint64_t) (chunk_n))((hg_uint64_t) (chunk_start))(
+                (hg_uint64_t) (chunk_end))((hg_uint64_t) (total_chunk_size))(
+                (hg_bulk_t) (bulk_handle)))
+
+MERCURY_GEN_PROC(
+        rpc_proxy_daemon_read_in_t,
+        ((hg_const_string_t) (path))((int64_t) (offset))(
+                (hg_uint64_t) (host_id))((hg_uint64_t) (host_size))(
+                (hg_uint64_t) (chunk_n))((hg_uint64_t) (chunk_start))(
+                (hg_uint64_t) (chunk_end))((hg_uint64_t) (total_chunk_size))(
+                (hg_bulk_t) (bulk_handle)))
+
+MERCURY_GEN_PROC(rpc_proxy_test_in_t, ((hg_const_string_t) (path)))
+
+MERCURY_GEN_PROC(rpc_proxy_get_dirents_in_t,
+                 ((hg_const_string_t) (path))((int32_t) (server))(
+                         (hg_bulk_t) (bulk_handle)))
+
+
 #endif // LFS_RPC_TYPES_HPP

include/config.hpp

@@ -35,9 +35,13 @@
 #define CLIENT_ENV_PREFIX "LIBGKFS_"
 #define DAEMON_ENV_PREFIX "GKFS_DAEMON_"
 #define COMMON_ENV_PREFIX "GKFS_"
+#define PROXY_ENV_PREFIX  "GKFS_PROXY_"
 
 namespace gkfs::config {
 
+// writes to dev null instead of chunk space, read is reading /dev/zero
+constexpr bool limbo_mode = false;
+
 constexpr auto hostfile_path = "./gkfs_hosts.txt";
 // We do not default this, ENV variable always required.
 constexpr auto forwarding_file_path = "";
@@ -48,14 +52,28 @@ namespace io {
  * If buffer is not zeroed, sparse regions contain invalid data.
  */
 constexpr auto zero_buffer_before_read = false;
+/*
+ * When the daemon handler serves a read request, it starts tasklets (for each
+ * chunk) from the io pool to read all chunks of that read request in parallel.
+ * Then another thread is waiting for the first tasklet to finish before
+ * initiating the bulk transfer back to the client for this chunk.
+ * This will continue in sequence, allowing gaps between bulk transfers while
+ * waiting. Although this is CPU efficient, it does not provide the highest I/O.
+ * if spin_lock_read is enabled it will try all tasklets if they are finished
+ * regardless of their order minimizing the gap between bulk transfers.
+ * Due to spinning in a loop this increases CPU utilization
+ */
+constexpr auto spin_lock_read = true;
 } // namespace io
 
 namespace log {
 constexpr auto client_log_path = "/tmp/gkfs_client.log";
 constexpr auto daemon_log_path = "/tmp/gkfs_daemon.log";
+constexpr auto proxy_log_path = "/tmp/gkfs_proxy.log";
 
 constexpr auto client_log_level = "info,errors,critical,hermes";
 constexpr auto daemon_log_level = 4; // info
+constexpr auto proxy_log_level = 4;  // info
 } // namespace log
 
 namespace metadata {
@@ -91,6 +109,20 @@ namespace data {
 constexpr auto chunk_dir = "chunks";
 } // namespace data
 
+namespace proxy {
+constexpr auto pid_path = "/tmp/gkfs_proxy.pid";
+constexpr auto fwd_create = true;
+constexpr auto fwd_stat = true;
+constexpr auto fwd_remove = true;
+constexpr auto fwd_update_size = true;
+constexpr auto fwd_io = true;
+constexpr auto fwd_chunk_stat = true;
+constexpr auto fwd_get_dirents_single = true;
+// Only use proxy for io if write/read size is higher than set value
+constexpr auto fwd_io_count_threshold = 0;
+
+} // namespace proxy
+
 namespace rpc {
 constexpr auto chunksize = 524288; // in bytes (e.g., 524288 == 512KB)
 // size of preallocated buffer to hold directory entries in rpc call
@@ -103,6 +135,8 @@ constexpr auto dirents_buff_size = (8 * 1024 * 1024); // 8 mega
 constexpr auto daemon_io_xstreams = 8;
 // Number of threads used for RPC handlers at the daemon
 constexpr auto daemon_handler_xstreams = 4;
+// Number of threads used for RPC handlers at the proxy
+constexpr auto proxy_handler_xstreams = 3;
 } // namespace rpc
 
 namespace rocksdb {

include/daemon/classes/fs_data.hpp

@@ -69,7 +69,10 @@ private:
 
     // RPC management
     std::string rpc_protocol_{};
+    std::string proxy_rpc_protocol_{};
     std::string bind_addr_{};
+    std::string bind_proxy_addr_{}; // optional when used with running proxy.
+                                    // Remains empty if unused
     std::string hosts_file_{};
     bool use_auto_sm_;
 
@@ -172,12 +175,23 @@ public:
     void
     rpc_protocol(const std::string& rpc_protocol);
 
+    const std::string&
+    proxy_rpc_protocol() const;
+    void
+    proxy_rpc_protocol(const std::string& proxy_rpc_protocol);
+
     const std::string&
     bind_addr() const;
 
     void
     bind_addr(const std::string& addr);
 
+    const std::string&
+    bind_proxy_addr() const;
+
+    void
+    bind_proxy_addr(const std::string& proxy_addr);
+
     const std::string&
     hosts_file() const;
 

include/daemon/classes/rpc_data.hpp

@@ -49,11 +49,13 @@ private:
     // Margo IDs. They can also be used to retrieve the Mercury classes and
     // contexts that were created at init time
     margo_instance_id server_rpc_mid_;
+    margo_instance_id proxy_server_rpc_mid_;
 
     // Argobots I/O pools and execution streams
     ABT_pool io_pool_;
     std::vector<ABT_xstream> io_streams_;
     std::string self_addr_str_;
+    std::string self_proxy_addr_str_;
     // Distributor
     std::shared_ptr<gkfs::rpc::Distributor> distributor_;
 
@@ -77,6 +79,12 @@ public:
     void
     server_rpc_mid(margo_instance* server_rpc_mid);
 
+    margo_instance*
+    proxy_server_rpc_mid();
+
+    void
+    proxy_server_rpc_mid(margo_instance* proxy_server_rpc_mid);
+
     ABT_pool
     io_pool() const;
 
@@ -93,7 +101,13 @@ public:
     self_addr_str() const;
 
     void
-    self_addr_str(const std::string& addr_stra);
+    self_addr_str(const std::string& addr_str);
+
+    const std::string&
+    self_proxy_addr_str() const;
+
+    void
+    self_proxy_addr_str(const std::string& proxy_addr_str);
 
     const std::shared_ptr<gkfs::rpc::Distributor>&
     distributor() const;

include/daemon/handler/rpc_defs.hpp

@@ -37,8 +37,7 @@ extern "C" {
 #include <margo.h>
 }
 
-/* visible API for RPC operations */
-
+// client <-> daemon RPCs
 DECLARE_MARGO_RPC_HANDLER(rpc_srv_get_fs_config)
 
 DECLARE_MARGO_RPC_HANDLER(rpc_srv_create)
@@ -64,7 +63,6 @@ DECLARE_MARGO_RPC_HANDLER(rpc_srv_mk_symlink)
 
 #endif
 
-
 // data
 DECLARE_MARGO_RPC_HANDLER(rpc_srv_remove_data)
 
@@ -76,4 +74,14 @@ DECLARE_MARGO_RPC_HANDLER(rpc_srv_truncate)
 
 DECLARE_MARGO_RPC_HANDLER(rpc_srv_get_chunk_stat)
 
+// proxy <-> daemon RPCs
+DECLARE_MARGO_RPC_HANDLER(rpc_srv_proxy_write)
+
+DECLARE_MARGO_RPC_HANDLER(rpc_srv_proxy_read)
+
+// client <-> proxy RPCs
+DECLARE_MARGO_RPC_HANDLER(proxy_rpc_srv_read)
+
+DECLARE_MARGO_RPC_HANDLER(proxy_rpc_srv_write)
+
 #endif // GKFS_DAEMON_RPC_DEFS_HPP