.gitlab-ci.yml

@@ -56,7 +56,8 @@ compile GekkoFS:
       -DCMAKE_BUILD_TYPE=Debug
       -DGKFS_BUILD_TESTS:BOOL=ON
       -DGKFS_INSTALL_TESTS:BOOL=ON
-      -DRPC_PROTOCOL="ofi+sockets"
+      -DGKFS_ENABLE_FORWARDING:BOOL=ON
+      -DGKFS_ENABLE_AGIOS:BOOL=ON
       -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH}
       -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}
       ${CI_PROJECT_DIR}

CMake/FindAGIOS.cmake

+find_path(AGIOS_INCLUDE_DIR
+    NAMES agios.h
+)
+
+find_library(AGIOS_LIBRARY
+    NAMES agios
+)
+
+set(AGIOS_INCLUDE_DIRS ${AGIOS_INCLUDE_DIR})
+set(AGIOS_LIBRARIES ${AGIOS_LIBRARY})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(AGIOS DEFAULT_MSG AGIOS_LIBRARIES AGIOS_INCLUDE_DIRS)
+
+mark_as_advanced(
+    AGIOS_LIBRARY
+    AGIOS_INCLUDE_DIR
+)
\ No newline at end of file

CMakeLists.txt

@@ -101,19 +101,6 @@ find_package(Threads REQUIRED)
 
 find_package(Date REQUIRED)
 
-set(RPC_PROTOCOL "ofi+sockets" CACHE STRING "Communication plugin used for RPCs")
-set_property(CACHE RPC_PROTOCOL PROPERTY STRINGS
-   "bmi+tcp"
-   "ofi+sockets"
-   "ofi+tcp"
-   "ofi+verbs"
-   "ofi+psm2"
-)
-message(STATUS "[gekkofs] RPC protocol: '${RPC_PROTOCOL}'")
-
-option(USE_SHM "Use shared memory for intra-node communication" OFF)
-message(STATUS "[gekkofs] Shared-memory communication: ${USE_SHM}")
-
 option(CREATE_CHECK_PARENTS "Check parent directory existance before creating child node" ON)
 message(STATUS "[gekkofs] Create checks parents: ${CREATE_CHECK_PARENTS}")
 
@@ -142,6 +129,13 @@ if(ENABLE_CLIENT_LOG)
 endif()
 message(STATUS "[gekkofs] Client logging output: ${ENABLE_CLIENT_LOG}")
 
+option(GKFS_ENABLE_FORWARDING "Enable forwarding mode" OFF)
+option(GKFS_ENABLE_AGIOS "Enable AGIOS scheduling library" OFF)
+if (GKFS_ENABLE_AGIOS)
+    find_package(AGIOS REQUIRED)
+endif ()
+
+
 set(CLIENT_LOG_MESSAGE_SIZE 1024 CACHE STRING "Maximum size of a log message in the client library")
 add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${CLIENT_LOG_MESSAGE_SIZE})
 message(STATUS "[gekkofs] Maximum log message size in the client library: ${CLIENT_LOG_MESSAGE_SIZE}")
@@ -232,6 +226,15 @@ if(GKFS_BUILD_TESTS)
     message(STATUS "[gekkofs] Preparing tests...")
     set(GKFS_TESTS_INTERFACE "lo" CACHE STRING "Network interface to use when running tests (default: lo)")
     message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}")
+
+    message(STATUS "[gekkofs] Check for forwarding tests...")
+    if(ENABLE_FORWARDING)
+        set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
+    else()
+        set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
+    endif()
+    message(STATUS "[gekkofs] Forwarding tests: ${GKFS_TESTS_FORWARDING}")
+
     add_subdirectory(tests)
 else()
     unset(GKFS_TESTS_INTERFACE CACHE)

README.md

 # GekkoFS
-This is a file system.
+GekkoFS is a file system capable of aggregating the local I/O capacity and performance of each compute node
+in a HPC cluster to produce a high-performance storage space that can be accessed in a distributed manner.
+This storage space allows HPC applications and simulations to run in isolation from each other with regards 
+to I/O, which reduces interferences and improves performance.
 
 # Dependencies
 
-## Rocksdb
+- Upgrade your gcc to version at least 4.8 to get C++11 support
+- CMake >3.6 (>3.11 for GekkoFS testing)
 
-### Debian/Ubuntu - Dependencies
+## Debian/Ubuntu - Dependencies
 
-- Upgrade your gcc to version at least 4.8 to get C++11 support.
-- Install snappy. This is usually as easy as: `sudo apt-get install libsnappy-dev`
-- Install zlib. Try: `sudo apt-get install zlib1g-dev`
-- Install bzip2: `sudo apt-get install libbz2-dev`
-- Install zstandard: `sudo apt-get install libzstd-dev`
-- Install lz4 `sudo apt-get install liblz4-dev`
+- snappy: `sudo apt-get install libsnappy-dev`
+- zlib: `sudo apt-get install zlib1g-dev`
+- bzip2: `sudo apt-get install libbz2-dev`
+- zstandard: `sudo apt-get install libzstd-dev`
+- lz4: `sudo apt-get install liblz4-dev`
+- uuid: `sudo apt-get install uuid-dev`
+- capstone: `sudo apt-get install libcapstone-dev`
 
 ### CentOS/Red Hat - Dependencies
-- Upgrade your gcc to version at least 4.8 to get C++11 support: yum install gcc48-c++
-- Install snappy:
-    `sudo yum install snappy snappy-devel`
-- Install zlib:
-    `sudo yum install zlib zlib-devel`
-- Install bzip2:
-    `sudo yum install bzip2 bzip2-devel`
-- Install ASAN (optional for debugging):
-    `sudo yum install libasan`
-- Install zstandard:
 
+
+- snappy: `sudo yum install snappy snappy-devel`
+- zlib: `sudo yum install zlib zlib-devel`
+- bzip2: `sudo yum install bzip2 bzip2-devel`
+- zstandard: 
 ```bash
    wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
    mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
@@ -33,18 +33,22 @@ This is a file system.
    cd zstd-1.1.3
    make && sudo make install
 ```
+- lz4: `sudo yum install lz4 lz4-devel`
+- uuid: `sudo yum install libuuid-devel`
+- capstone: `sudo yum install capstone capstone-devel`
+
 
 # Usage
 
 ## Clone and compile direct GekkoFS dependencies
 
-- Go to the `scripts` folder and first clone all dependencies projects. You can choose the according na_plugin
+- Go to the `scripts` folder and first clone all dependencies projects. You can choose the according network (na) plugin
 (execute the script for help):
 
 ```bash
-usage: dl_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CLUSTER>] [-d <DEPENDENCY>] 
+usage: dl_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CONFIG>] [-d <DEPENDENCY>]
                     source_path
-	
+
 
 This script gets all GekkoFS dependency sources (excluding the fs itself)
 
@@ -55,29 +59,32 @@ positional arguments:
 optional arguments:
         -h, --help              shows this help message and exits
         -l, --list-dependencies
-                                list dependencies available for download
+                                list dependencies available for download with descriptions
         -n <NAPLUGIN>, --na <NAPLUGIN>
                                 network layer that is used for communication. Valid: {bmi,ofi,all}
-                                defaults to 'all'
-        -c <CLUSTER>, --cluster <CLUSTER>
-                                additional configurations for specific compute clusters
-                                supported clusters: {mogon1,mogon2,fh2}
+                                defaults to 'ofi'
+        -c <CONFIG>, --config <CONFIG>
+                                allows additional configurations, e.g., for specific clusters
+                                supported values: {mogon2, mogon1, ngio, direct, all}
+                                defaults to 'direct'
         -d <DEPENDENCY>, --dependency <DEPENDENCY>
-                                download a specific dependency. If unspecified 
-                                all dependencies are built and installed.
+                                download a specific dependency and ignore --config setting. If unspecified
+                                all dependencies are built and installed based on set --config setting.
+                                Multiple dependencies are supported: Pass a space-separated string (e.g., "ofi mercury"
+        -v, --verbose           Increase download verbosity
 ```
 - Now use the install script to compile them and install them to the desired directory. You can choose the according
-na_plugin (execute the script for help):
+(na) network plugin (execute the script for help):
 
 ```bash
-usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CLUSTER>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>]
+usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CONFIG>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>]
                       source_path install_path
-	
+
 
 This script compiles all GekkoFS dependencies (excluding the fs itself)
 
 positional arguments:
-    source_path 	path to the cloned dependencies path from clone_dep.sh
+    source_path         path to the cloned dependencies path from clone_dep.sh
     install_path    path to the install path of the compiled dependencies
 
 
@@ -88,12 +95,14 @@ optional arguments:
     -n <NAPLUGIN>, --na <NAPLUGIN>
                 network layer that is used for communication. Valid: {bmi,ofi,all}
                 defaults to 'all'
-    -c <CLUSTER>, --cluster <CLUSTER>
-                additional configurations for specific compute clusters
-                supported clusters: {mogon1,mogon2,fh2}
+    -c <CONFIG>, --config <CONFIG>
+                allows additional configurations, e.g., for specific clusters
+                supported values: {mogon1, mogon2, ngio, direct, all}
+                defaults to 'direct'
     -d <DEPENDENCY>, --dependency <DEPENDENCY>
-                build and install a specific dependency. If unspecified 
-                all dependencies are built and installed.
+                download a specific dependency and ignore --config setting. If unspecified
+                all dependencies are built and installed based on set --config setting.
+                Multiple dependencies are supported: Pass a space-separated string (e.g., "ofi mercury"
     -j <COMPILE_CORES>, --compilecores <COMPILE_CORES>
                 number of cores that are used to compile the dependencies
                 defaults to number of available cores
@@ -101,18 +110,14 @@ optional arguments:
 ```
 
 ## Compile GekkoFS
-You need to decide what Mercury NA plugin you want to use. The following NA plugins are available, although only BMI is considered stable at the moment.
- - `ofi+sockets` for using the libfabric plugin with TCP
- - `ofi+tcp` for using the libfabric plugin with TCP (new version)
- - `ofi+verbs` for using the libfabric plugin with Infiniband verbs (not threadsafe. Do not use.)
- - `ofi+psm2` for using the libfabric plugin with Intel Omni-Path
- - `bmi+tcp` for using the bmi plugin with the tcp protocol 
+
+If above dependencies have been installed outside of the usual system paths, use CMake's `-DCMAKE_PREFIX_PATH` to
+make this path known to CMake.
 
 ```bash
 mkdir build && cd build
-cmake -DCMAKE_BUILD_TYPE=Release -DRPC_PROTOCOL='ofi+sockets' ..
+cmake -DCMAKE_BUILD_TYPE=Release ..
 make
-make install
 ```
 
 In order to build self-tests, the *optional* GKFS_BUILD_TESTS CMake option needs
@@ -121,35 +126,67 @@ to be enabled when building. Once that is done, tests can be run by running
 
 ```bash
 mkdir build && cd build
-cmake -DGKFS_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Release -DRPC_PROTOCOL='ofi+sockets' ..
+cmake -DGKFS_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Release ..
 make
 make test
 make install
 ```
 
-**IMPORTANT:** Please note that the testing framework requires Python 3.6 as an
-additional dependency in order to run.
+**IMPORTANT:** Please note that the testing framework requires Python 3.6 and >CMake 3.11 as
+additional dependencies in order to run.
 
 ## Run GekkoFS
 
-First on each node a daemon has to be started. This can be done in two ways using the `gkfs_daemon` binary directly or
-the corresponding startup and shutdown scripts. The latter is recommended for cluster usage. It requires pssh (or
-parallel-ssh) with python2.
+On each node a daemon (`gkfs_daemon` binary) has to be started. Other tools can be used to execute
+the binary on many nodes, e.g., `srun`, `mpiexec/mpirun`, `pdsh`, or `pssh`.
+
+You need to decide what Mercury NA plugin you want to use for network communication. `ofi+sockets` is the default.
+The `-P` argument is used for setting another RPC protocol. See below.
+
+ - `ofi+sockets` for using the libfabric plugin with TCP (stable)
+ - `ofi+tcp` for using the libfabric plugin with TCP (slower than sockets)
+ - `ofi+verbs` for using the libfabric plugin with Infiniband verbs (reasonably stable)
+ - `ofi+psm2` for using the libfabric plugin with Intel Omni-Path (unstable)
+ - `bmi+tcp` for using the bmi plugin with TCP (alternative to libfabric)
 
 ### Start and shut down daemon directly
 
-`./build/bin/gkfs_daemon -r <fs_data_path> -m <pseudo_mount_dir_path>`
- 
-Shut it down by gracefully killing the process.
- 
-### Startup and shutdown scripts
+`./build/src/daemon/gkfs_daemon -r <fs_data_path> -m <pseudo_mount_dir_path>`
+
+Further options:
+```bash
+Allowed options:
+  -h [ --help ]             Help message
+  -m [ --mountdir ] arg     pseudo mountdir
+  -r [ --rootdir ] arg      data directory
+  -i [ --metadir ] arg      metadata directory, if not set rootdir is used for
+                            metadata
+  -l [ --listen ] arg       Address or interface to bind the daemon on.
+                            Default: local hostname.
+                            When used with ofi+verbs the FI_VERBS_IFACE
+                            environment variable is set accordingly which
+                            associates the verbs device with the network
+                            interface. In case FI_VERBS_IFACE is already
+                            defined, the argument is ignored. Default 'ib'.
+  -H [ --hosts-file ] arg   Shared file used by deamons to register their
+                            enpoints. (default './gkfs_hosts.txt')
+  -P [ --rpc_protocol ] arg Used RPC protocol for inter-node communication.
+                            Available: {ofi+sockets, ofi+verbs, ofi+psm2} for
+                            (TCP, Infiniband, and Omni-Path, respectively.
+                            (Default ofi+sockets)
+                            Libfabric must have verbs or psm2 support enabled.
+  --auto_sm                 Enables intra-node communication (IPCs) via the
+                            `na+sm` (shared memory) protocol, instead of using
+                            the RPC protocol. (Default off)
+  --version                 print version and exit
+```
 
-The scripts are located in `scripts/{startup_gkfs.py, shutdown_gkfs.py}`. Use the -h argument for their usage.
+Shut it down by gracefully killing the process (SIGTERM).
 
 ## Miscellaneous
 
 Metadata and actual data will be stored at the `<fs_data_path>`. The path where the application works on is set with
-`<pseudo_mount_dir_path>`
+`<pseudo_mount_dir_path>`.
  
 Run the application with the preload library: `LD_PRELOAD=<path>/build/lib/libgkfs_intercept.so ./application`. In the case of
 an MPI application use the `{mpirun, mpiexec} -x` argument.

config/GekkoFWD/agios/access_times.func

+2
+8	64
+566.8405	6784.471
+238.2483	1173.456
+145.6056	1018.143
+187.2437	-201.8793
+64	128
+571.2364	-740.0433
+234.6933	41.99119
+218.3765	-6295.179
+113.1743	2235.442

config/GekkoFWD/agios/agios.conf

+library_options:
+{  #ATTENTION: with my most recent modification on statistics, the prediction module is NOT WORKING, DO NOT USE IT.
+	#should we generate trace files during execution? 
+	trace = false ;
+
+	#should we trace predicted requests? (for debug purposes, this trace will not be useful for future executions)
+	trace_predict = false ;
+
+	#should we make a complete trace, with all scheduler's operations, such as waiting times? (for debug purposes)
+	trace_full = false ;
+
+	#should the prediction module read trace files? (if false, the Prediction Module is useless)
+	predict_read_traces = false ;
+
+	#should the prediction module try to predict aggregations? (requires predict_read_traces = true)
+	predict_request_aggregation = false ;
+
+	#should the prediction module create simplified traces with information (the metrics) it obtained from the real traces?
+	predict_write_simplified_traces = false;
+
+	#the tolerance for arrival times difference when checking if two predicted requests are the same (in %)
+	prediction_time_error = 10
+
+	#this parameter gives the frequency with which the prediction module will redo its predicted aggregations  (in number of requests that must be processed between refreshs). This is necessary because these predictions use a factor that represents the ability to overlap waiting times with processing of other requests. At initialization, this factor will be calculated from the provided trace files, but during execution it can be recalculated using measurements for this ability during the actual scheduling. If the parameter is set to -1, aggregations will not be recalculated during execution.  (in number of requests)
+	prediction_recalculate_alpha_period = -1
+
+	#prefix and sufix for trace files (with path). Their names must be trace_file_prefix+"."+number+"."+trace_file_sufix, with ordered numbers (no holes)
+	trace_file_prefix = "/tmp/agios_tracefile"
+	trace_file_sufix = "out"
+	#prefix for simple trace files (with path). Their names will be prefix+"."+number+"."+trace_file_sufix
+	simple_trace_prefix = "/tmp/agios_simpletracefile"
+
+	#parameters used by aIOLi and MLF
+	waiting_time = 900000
+	aioli_quantum = 65536
+	mlf_quantum = 8192
+
+	#parameter used by TW (ms)
+	time_window_size = 1000 #the paper proposing TW recommends 1000 for HDD and 250 for SSD.
+
+	#file (with path) with access times functions (generated by SeRRa - http://serratoool.bitbucket.org/). Used by aIOLi to quantum assignment and by the mechanism that automatically selects the best scheduling algorithm to use. If you are using a static algorithm which is not aIOLi, this does not matter, but you need to provide it anyway. In this case, you can use the one provided as example with the library source code 
+	access_times_func_file = "/tmp/access_times.func"
+
+	#to how many scheduling algorithms the performance module keeps measurements. When we are changing scheduling algorithms, we may observe new measurements (through the agios_release_request function) to the previous algorithms, so we could update information we have for them. It makes no sense to have a big value for performance_values if we don't change algorithms too often
+	performance_values = 5
+
+	#how many scheduling algorithms changes we register in the stats file?
+	proc_algs = 1000
+
+	#default I/O scheduling algorithm to use (the one to be used if the previous value was set to false)
+	#existing algorithms (case sensitive): "MLF", "aIOLi", "SJF", "TO", "TO-agg", "SRTF", "TW", "NOOP", "DYN_TREE", "ARMED_BANDIT" (case sensitive) 
+	# SRTF it's experimental and uses information from trace files (don't use it if these are not available)
+	# NOOP is the NO operation scheduling algorithm, you should not observe performance improvements by using this one
+	# TW only makes sense if the user is providing AGIOS correct application id for each request. Don't use it otherwise
+	# DYN_TREE is a dynamic scheduling algorithm which selects the best (among MLF, AIOLI, SFJ, TO, TO-AGG, and NOOP) according to a decision tree. Use it only when using AGIOS to schedule requests to parallel file system' servers.
+	# ARMED_BANDIT is another dynamic algorithm which periodically selects between MLF, aIOLi, SJF, TO, TO-agg, and NOOP. It keeps performance measurements for these algorithms and gives more probability to the better ones. 
+	default_algorithm = "TO";
+
+	# TWINS window size, in us
+	twins_window_duration = 1000; 
+	dynamic_twins = false;
+	# time between window size adaptation, in ms
+	dynamic_twins_period = 1000;
+	
+	dynamic_twins_clients = 32;
+	dynamic_twins_processes = 128;
+
+	# Only relevant if default_algorithm is a dynamic one. this parameter gives the frequency with which the automatic scheduling algorithm selection will recalculate the scheduling algorithm. This selection will be done using the access pattern from this period. If -1 is provided, then the selection will be done at the beginning of execution only (using information from traces for DYN_TREE). The next parameter gives the minimum number of requests which need to happen in this period for the selection to be done (otherwise we will wait longer before recalculating).  (in msec). Notice that it makes no sense to use ARMED_BANDIT without setting period and min_reqnumber, as it needs to be iteractive.
+	select_algorithm_period = 1000
+
+	select_algorithm_min_reqnumber=1
+
+	#also for dynamic algorithms. says if TW should be one of the options considered by the dynamic scheduler. TW requires identifying requests according to the applications they come from. If this is not possible, don't use TW!
+	enable_TW = false ;
+
+	#if default_algorithm is a dynamic algorithm, you need to indicate which static algorithm to use at first (before automatically selecting the next one). Only relevant if you are using ARMED_BANDIT
+	starting_algorithm = "SJF" ;
+
+	# ARMED_BANDIT parameters - only relevant if default_algorithm is ARMED_BANDIT, but need to be provided anyway. min_ab_probability gives the minimum probability given to scheduling algorithms which performed poorly. The algorithm needs to maintain some probability to accomodate changes in the access pattern. validity_window is the period of time (in msec) for which performance measurements are still valid, after this time period we discard them (so we adapt to new situations). performance_window determines how many performance measurements the ARMED_BANDIT algorithms keeps for each scheduling algorithm option (taking the average of them). Keeping a huge window takes more memory. Moreover, it is related with validity_window and select_algorithm_period as if we discard measurements too often, we will never fill the whole window.
+	min_ab_probability = 3 
+	validity_window = 360000
+	performance_window = 10
+};
+user_info:
+{
+	#stripe size used by the library's users (in bytes). This is used for detecting the access pattern at a parallel file system server. Useless for other uses. 
+	stripe_size = 32768 ;
+
+	#maximum buffer size used for storing trace parts (in KB). Having a buffer avoids generating requests to the local file system, which interfere in performance. On the other hand, having a large buffer can affect performance and decrease available space for data buffer.
+	max_trace_buffer_size = 32768 ;
+
+};

docker/build_env.docker

@@ -25,6 +25,7 @@ RUN yum -y -q update  && yum -y -q install \
 	gcc \
 	gcc-c++ \
 	openssl-devel \
+	libconfig-devel \
 	# Mercury dependencies
 	libtool \
 	libtool-ltdl-devel \
@@ -34,6 +35,7 @@ RUN yum -y -q update  && yum -y -q install \
 	snappy-devel \
 	libzstd-devel \
 	lz4-devel \
+	bzip2 \
 	bzip2-devel \
 	# ada-fs requires C++ 14
 	devtoolset-8-gcc \
@@ -81,6 +83,7 @@ RUN yum -y -q update  && yum -y -q install \
 	gcc \
 	gcc-c++ \
 	openssl-devel \
+	libconfig-devel \
 	# Mercury dependencies
 	libtool \
 	libtool-ltdl-devel \
@@ -90,6 +93,7 @@ RUN yum -y -q update  && yum -y -q install \
 	snappy-devel \
 	libzstd-devel \
 	lz4-devel \
+	bzip2 \
 	bzip2-devel \
 	# ada-fs requires C++ 14
 	devtoolset-8-gcc \

docker/debian_build_env.docker

@@ -19,6 +19,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 		cmake \
 		gcc \
 		g++ \
+		# AGIOS dependencies
+		libconfig-dev \
 		# Mercury dependencies
 		libltdl-dev \
 		lbzip2 \

include/client/env.hpp

@@ -33,6 +33,9 @@ static constexpr auto LOG_OUTPUT          = ADD_PREFIX("LOG_OUTPUT");
 static constexpr auto LOG_OUTPUT_TRUNC    = ADD_PREFIX("LOG_OUTPUT_TRUNC");
 static constexpr auto CWD                 = ADD_PREFIX("CWD");
 static constexpr auto HOSTS_FILE          = ADD_PREFIX("HOSTS_FILE");
+#ifdef GKFS_ENABLE_FORWARDING
+static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE");
+#endif
 
 } // namespace env
 } // namespace gkfs

include/client/gkfs_functions.hpp

@@ -42,7 +42,10 @@ int gkfs_stat(const std::string& path, struct stat* buf, bool follow_links = tru
 // Implementation of statx, it uses the normal stat and maps the information to the statx structure
 // Follow links is true by default 
 #ifdef STATX_TYPE
-int gkfs_statx(int dirfd, const std::string& path, int flags, unsigned int mask,struct statx* buf, bool follow_links = true );
+
+int gkfs_statx(int dirfd, const std::string& path, int flags, unsigned int mask, struct statx* buf,
+               bool follow_links = true);
+
 #endif
 
 int gkfs_statfs(struct statfs* buf);

include/client/hooks.hpp

@@ -44,10 +44,10 @@ int hook_read(unsigned int fd, void* buf, size_t count);
 
 int hook_pread(unsigned int fd, char* buf, size_t count, loff_t pos);
 
-int hook_readv(unsigned long fd, const struct iovec * iov, unsigned long iovcnt);
+int hook_readv(unsigned long fd, const struct iovec* iov, unsigned long iovcnt);
 
-int hook_preadv(unsigned long fd, const struct iovec * iov, unsigned long iovcnt,
-                 unsigned long pos_l, unsigned long pos_h);
+int hook_preadv(unsigned long fd, const struct iovec* iov, unsigned long iovcnt,
+                unsigned long pos_l, unsigned long pos_h);
 
 int hook_write(unsigned int fd, const char* buf, size_t count);
 

include/client/preload_context.hpp

@@ -82,6 +82,9 @@ private:
 
     std::vector<hermes::endpoint> hosts_;
     uint64_t local_host_id_;
+    uint64_t fwd_host_id_;
+    std::string rpc_protocol_;
+    bool auto_sm_{false};
 
     bool interception_enabled_;
 
@@ -118,11 +121,22 @@ public:
 
     void clear_hosts();
 
-
     uint64_t local_host_id() const;
 
     void local_host_id(uint64_t id);
 
+    uint64_t fwd_host_id() const;
+
+    void fwd_host_id(uint64_t id);
+
+    const std::string& rpc_protocol() const;
+
+    void rpc_protocol(const std::string& rpc_protocol);
+
+    bool auto_sm() const;
+
+    void auto_sm(bool auto_sm);
+
     RelativizeStatus relativize_fd_path(int dirfd,
                                         const char* raw_path,
                                         std::string& relative_path,

include/client/preload_util.hpp

@@ -59,10 +59,14 @@ std::shared_ptr<gkfs::metadata::Metadata> get_metadata(const std::string& path,
 
 int metadata_to_stat(const std::string& path, const gkfs::metadata::Metadata& md, struct stat& attr);
 
-std::vector<std::pair<std::string, std::string>> load_hostfile(const std::string& lfpath);
-
 void load_hosts();
 
+void load_forwarding_map();
+
+std::vector<std::pair<std::string, std::string>> read_hosts_file();
+
+void connect_to_hosts(const std::vector<std::pair<std::string, std::string>>& hosts);
+
 } // namespace util
 } // namespace gkfs
 

include/client/rpc/forward_data.hpp

@@ -24,14 +24,16 @@ struct ChunkStat {
     unsigned long chunk_free;
 };
 
-ssize_t forward_write(const std::string& path, const void* buf, bool append_flag, off64_t in_offset,
-                      size_t write_size, int64_t updated_metadentry_size);
+// TODO once we have LEAF, remove all the error code returns and throw them as an exception.
 
-ssize_t forward_read(const std::string& path, void* buf, off64_t offset, size_t read_size);
+std::pair<int, ssize_t> forward_write(const std::string& path, const void* buf, bool append_flag, off64_t in_offset,
+                                      size_t write_size, int64_t updated_metadentry_size);
+
+std::pair<int, ssize_t> forward_read(const std::string& path, void* buf, off64_t offset, size_t read_size);
 
 int forward_truncate(const std::string& path, size_t current_size, size_t new_size);
 
-ChunkStat forward_get_chunk_stat();
+std::pair<int, ChunkStat> forward_get_chunk_stat();
 
 } // namespace rpc
 } // namespace gkfs

include/client/rpc/forward_metadata.hpp

@@ -16,6 +16,7 @@
 #define GEKKOFS_CLIENT_FORWARD_METADATA_HPP
 
 #include <string>
+#include <memory>
 
 /* Forward declaration */
 namespace gkfs {
@@ -24,9 +25,12 @@ class OpenDir;
 }
 namespace metadata {
 struct MetadentryUpdateFlags;
+
 class Metadata;
 }
 
+// TODO once we have LEAF, remove all the error code returns and throw them as an exception.
+
 namespace rpc {
 
 int forward_create(const std::string& path, mode_t mode);
@@ -40,12 +44,12 @@ int forward_decr_size(const std::string& path, size_t length);
 int forward_update_metadentry(const std::string& path, const gkfs::metadata::Metadata& md,
                               const gkfs::metadata::MetadentryUpdateFlags& md_flags);
 
-int forward_update_metadentry_size(const std::string& path, size_t size, off64_t offset, bool append_flag,
-                                   off64_t& ret_size);
+std::pair<int, off64_t>
+forward_update_metadentry_size(const std::string& path, size_t size, off64_t offset, bool append_flag);
 
-int forward_get_metadentry_size(const std::string& path, off64_t& ret_size);
+std::pair<int, off64_t> forward_get_metadentry_size(const std::string& path);
 
-void forward_get_dirents(gkfs::filemap::OpenDir& open_dir);
+std::pair<int, std::shared_ptr<gkfs::filemap::OpenDir>> forward_get_dirents(const std::string& path);
 
 #ifdef HAS_SYMLINKS
 

include/config.hpp

@@ -24,6 +24,7 @@ namespace gkfs {
 namespace config {
 
 constexpr auto hostfile_path = "./gkfs_hosts.txt";
+constexpr auto forwarding_file_path = "./gkfs_forwarding.map";
 
 namespace io {
 /*

include/daemon/classes/fs_data.hpp

@@ -38,7 +38,7 @@ class FsData {
 private:
     FsData() {}
 
-    //logger
+    // logger
     std::shared_ptr<spdlog::logger> spdlogger_;
 
     // paths
@@ -46,8 +46,11 @@ private:
     std::string mountdir_;
     std::string metadir_;
 
+    // RPC management
+    std::string rpc_protocol_;
     std::string bind_addr_;
     std::string hosts_file_;
+    bool use_auto_sm_;
 
     // Database
     std::shared_ptr<gkfs::metadata::MetadataDB> mdb_;
@@ -99,12 +102,20 @@ public:
 
     void storage(const std::shared_ptr<gkfs::data::ChunkStorage>& storage);
 
+    const std::string& rpc_protocol() const;
+
+    void rpc_protocol(const std::string& rpc_protocol);
+
     const std::string& bind_addr() const;
 
     void bind_addr(const std::string& addr);
 
     const std::string& hosts_file() const;
 
+    bool use_auto_sm() const;
+
+    void use_auto_sm(bool use_auto_sm);
+
     void hosts_file(const std::string& lookup_file);
 
     bool atime_state() const;

include/daemon/scheduler/agios.hpp

+#ifndef IFS_SCHEDULER_HPP
+#define IFS_SCHEDULER_HPP
+
+#include <agios.h>
+
+void agios_initialize();
+
+void agios_shutdown();
+
+void* agios_callback(int64_t request_id);
+
+void* agios_callback_aggregated(int64_t* requests, int32_t total);
+
+void* agios_eventual_callback(int64_t request_id, void* info);
+
+unsigned long long int generate_unique_id();
+
+#endif
\ No newline at end of file

include/global/cmake_configure.hpp.in

@@ -14,8 +14,6 @@
 #ifndef GKFS_CMAKE_CONFIGURE_HPP
 #define GKFS_CMAKE_CONFIGURE_HPP
 
-#define RPC_PROTOCOL "@RPC_PROTOCOL@"
-#cmakedefine01 USE_SHM
 #cmakedefine01 CREATE_CHECK_PARENTS
 #cmakedefine01 LOG_SYSCALLS
 

include/global/global_defs.hpp

@@ -46,6 +46,7 @@ constexpr auto ofi_psm2 = "ofi+psm2";
 constexpr auto ofi_sockets = "ofi+sockets";
 constexpr auto ofi_tcp = "ofi+tcp";
 constexpr auto ofi_verbs = "ofi+verbs";
+constexpr auto na_sm = "na+sm";
 } // namespace protocol
 } // namespace rpc