.gitlab-ci.yml

@@ -65,23 +65,6 @@ gkfs:
       - ${INSTALL_PATH}
     expire_in: 1 week
 
-gkfwd:
-  stage: build
-  image: gekkofs/deps:0.9.2
-  interruptible: true
-  needs: []
-  script:
-    - cmake --preset ci-forwarding-coverage
-    - cmake --build ${BUILD_PATH} -j $(nproc) --target install
-    # reduce artifacts size
-    - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH}
-  artifacts:
-    paths:
-      - ${BUILD_PATH}
-      - ${INSTALL_PATH}
-    expire_in: 1 week
-
-
 ################################################################################
 ## Testing
 ################################################################################
@@ -156,12 +139,12 @@ gkfwd:integration:
   stage: test
   image: gekkofs/testing:0.9.2
   interruptible: true
-  needs: ['gkfwd']
+  needs: ['gkfs']
   parallel:
     matrix:
       - SUBTEST: [ forwarding ]
-  rules:
-    - when: never
+ # rules:
+ #   - when: never
 
   script:
     ## run tests
@@ -274,7 +257,7 @@ coverage:baseline:
   stage: report
   image: gekkofs/testing:0.9.2
   interruptible: true
-  needs: ['gkfs', 'gkfwd']
+  needs: ['gkfs']
 
   script:
     ## capture initial coverage information to establish a baseline
@@ -301,7 +284,7 @@ coverage:
   image: gekkofs/testing:0.9.2
 #  needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration',
 #           'gkfs:unit' ]
-  needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit' ]
+  needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration']
   script:
     - cd ${CI_PROJECT_DIR}
     - cmake

CHANGELOG.md

@@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+- Replication without using the server. NUM_REPL (0 < NUM_REPL < num_servers) env variable defines the number of 
+replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)).
+- Modified write and reads to use a bitset instead of the traditional hash per chunk in the server.
+- Added reattemp support in get_fs_config to other servers, when the initial server fails.
+- Fused GekkoFWD and GekkoFS. GekkoFWD is enabled with the `--enable-following` in the server configuration and the ENV variable
+`LIBGKFS_FORWARDING_MAP_FILE` in the clients. ([!170](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/170)).
 
 ### New
 

CMake/gkfs-options.cmake

@@ -348,14 +348,6 @@ gkfs_define_option(
 # I/O forwarding
 ################################################################################
 
-## Forwarding support
-gkfs_define_option(
-  GKFS_ENABLE_FORWARDING
-  HELP_TEXT "Enable I/O forwarding mode"
-  DEFAULT_VALUE OFF
-  DESCRIPTION "Use ${PROJECT_NAME} as an I/O forwarding layer"
-)
-
 ## Scheduling in I/O forwarding mode
 gkfs_define_option(
   GKFS_ENABLE_AGIOS

CMakeLists.txt

@@ -32,16 +32,17 @@ cmake_minimum_required(VERSION 3.13)
 project(
     GekkoFS
     VERSION 0.9.1
+    LANGUAGES ASM CXX C
 )
 
 enable_testing()
 
-if (NOT CMAKE_COMPILER_IS_GNUCC)
-    message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported")
-endif ()
-if (NOT CMAKE_COMPILER_IS_GNUCXX)
-    message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported")
-endif ()
+#if (NOT CMAKE_COMPILER_IS_GNUCC)
+ #   message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported")
+#endif ()
+#if (NOT CMAKE_COMPILER_IS_GNUCXX)
+#    message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported")
+#endif ()
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -338,11 +339,7 @@ if (GKFS_BUILD_TESTS)
     message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}")
 
     message(STATUS "[gekkofs] Check for forwarding tests...")
-    if (GKFS_ENABLE_FORWARDING)
-        set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
-    else ()
-        set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
-    endif ()
+    set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
     message(STATUS "[gekkofs] Forwarding tests: ${GKFS_TESTS_FORWARDING}")
     message(STATUS "[gekkofs] Check for guided distributor tests...")
     if (GKFS_USE_GUIDED_DISTRIBUTION)

CMakePresets.json

@@ -27,14 +27,6 @@
         "deprecated": true
       }
     },
-    {
-      "name": "forwarding",
-      "inherits": "default",
-      "hidden": true,
-      "cacheVariables": {
-        "GKFS_ENABLE_FORWARDING": true
-      }
-    },
     {
       "name": "release",
       "hidden": true,
@@ -113,30 +105,6 @@
         "release"
       ]
     },
-    {
-      "name": "forwarding-debug",
-      "displayName": "Forwarding gekkofs (debug)",
-      "inherits": [
-        "forwarding",
-        "debug"
-      ]
-    },
-    {
-      "name": "forwarding-coverage",
-      "displayName": "Forwarding gekkofs (coverage)",
-      "inherits": [
-        "forwarding",
-        "coverage"
-      ]
-    },
-    {
-      "name": "forwarding-release",
-      "displayName": "Forwarding gekkofs (release)",
-      "inherits": [
-        "forwarding",
-        "release"
-      ]
-    },
     {
       "name": "ci-debug",
       "displayName": "Default gekkofs (debug, CI flags)",
@@ -171,33 +139,6 @@
         "default",
         "release"
       ]
-    },
-    {
-      "name": "ci-forwarding-debug",
-      "displayName": "Forwarding gekkofs (debug, CI flags)",
-      "inherits": [
-        "ci",
-        "forwarding",
-        "debug"
-      ]
-    },
-    {
-      "name": "ci-forwarding-coverage",
-      "displayName": "Forwarding gekkofs (coverage, CI flags)",
-      "inherits": [
-        "ci",
-        "forwarding",
-        "coverage"
-      ]
-    },
-    {
-      "name": "ci-forwarding-release",
-      "displayName": "Forwarding gekkofs (release, CI flags)",
-      "inherits": [
-        "ci",
-        "forwarding",
-        "release"
-      ]
     }
   ]
 }

README.md

@@ -319,6 +319,12 @@ Support for fstat in renamed files is included.
 
 This is disabled by default.
 
+### Replication
+The user can enable the data replication feature by setting the replication environment variable:
+`LIBGKFS_NUM_REPL=<num repl>`.
+The number of replicas should go from 0 to the number of servers-1. 
+The replication environment variable can be set up for each client, independently.
+
 ## Acknowledgment
 
 This software was partially supported by the EC H2020 funded NEXTGenIO project (Project ID: 671951, www.nextgenio.eu).

docs/sphinx/users/dependencies.rst

@@ -43,4 +43,4 @@ Required
 Optional
 --------
 
-- `agios <https://github.com/francielizanon/agios>`_ (commit c26a654 or newer) to enable GekkoFWD mode.
+- `agios <https://github.com/francielizanon/agios>`_ (commit c26a654 or newer) to enable GekkoFWD mode scheduling.

docs/sphinx/users/forwarding.rst

@@ -24,13 +24,11 @@ PFS for storage, instead of a local store available at the compute nodes.
 Enabling GekkoFWD
 ------------------
 
-To enable the I/O forwarding mode of GekkoFS, the
-:code:`GKFS_ENABLE_FORWARDING` CMake option should be enabled, :ref:`when
-configuring <building_gekkofs>` the build:
-
-.. code-block:: console
-
-    $ cmake -DENABLE_FORWARDING:BOOL=ON
+To enable the I/O forwarding mode of GekkoFS, an environment 
+variable named :code:`LIBGKFS_FORWARDING_MAP_FILE` is provided
+to allow users to identify the map file on each client.
+This environment variable will enable GekkoFS forwarding, then we 
+only need to use :code:`--enable-forwarding`` option in the servers.
 
 I/O Scheduling
 --------------

examples/gfind/CMakeLists.txt

@@ -28,9 +28,17 @@
 
 set (CMAKE_CXX_STANDARD 14)
 add_executable(sfind sfind.cpp)
+add_executable(gkfs_lib_example gkfs_lib_example.cpp)
+
+target_link_libraries(gkfs_lib_example
+  PRIVATE gkfs_user_lib
+)
 
 if(GKFS_INSTALL_TESTS)
     install(TARGETS sfind 
         DESTINATION ${CMAKE_INSTALL_BINDIR}
     )
+    install(TARGETS gkfs_lib_example
+        DESTINATION ${CMAKE_INSTALL_BINDIR}
+    )
 endif()

examples/gfind/gkfs_lib_example.cpp

+/*
+  Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain
+  Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany
+
+  This software was partially supported by the
+  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+  This software was partially supported by the
+  ADA-FS project under the SPPEXA project funded by the DFG.
+
+  This file is part of GekkoFS.
+
+  GekkoFS is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  GekkoFS is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.
+
+  SPDX-License-Identifier: GPL-3.0-or-later
+*/
+
+
+#include <cmath>
+#include <cstring>
+#include <getopt.h>
+#include <iostream>
+#include <queue>
+#include <regex.h>
+#include <stdio.h>
+#include <string>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits>
+#include <cstdint>
+#include <client/gkfs_functions.hpp>
+
+using namespace std;
+
+/* Function exported from GekkoFS LD_PRELOAD, code needs to be compiled with
+ * -fPIC */
+extern "C" int
+gkfs_init() __attribute__((weak));
+
+extern "C" int
+gkfs_end() __attribute__((weak));
+
+void
+init_preload(){};
+void
+destroy_preload(){};
+
+void
+write_file(std::string filename) {
+    // Open File
+    int fd = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDWR | O_CREAT);
+
+    cout << "FD open  " << fd << endl;
+    char* buf = "testing";
+
+    int size = gkfs::syscall::gkfs_write(fd, buf, 7);
+
+    cout << "FD size" << size << endl;
+
+    gkfs::syscall::gkfs_close(fd);
+}
+
+
+void
+read_file(std::string filename) {
+    int fdread = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDONLY);
+    if(fdread == -1)
+        return;
+    char* bufread = (char*) malloc(10);
+    int sizeread = gkfs::syscall::gkfs_read(fdread, bufread, 7);
+
+    cout << "Reading : " << sizeread << " --> " << bufread << endl;
+
+    gkfs::syscall::gkfs_close(fdread);
+}
+int
+main(int argc, char** argv) {
+    cout << "GekkoFS Client library test" << endl;
+
+    auto res = gkfs_init();
+
+    cout << "Init result " << res << endl;
+
+    write_file("/test.tmp");
+
+    read_file("/test.tmp");
+
+    res = gkfs_end();
+
+    cout << "End result " << res << endl;
+}

include/CMakeLists.txt

@@ -34,7 +34,3 @@ add_subdirectory(daemon)
 add_subdirectory(client)
 
 target_sources(gkfs_daemon PUBLIC config.hpp version.hpp.in)
-
-if(GKFS_ENABLE_FORWARDING)
-  target_sources(gkfwd_daemon PUBLIC config.hpp version.hpp.in)
-endif()

include/client/CMakeLists.txt

@@ -54,30 +54,29 @@ target_sources(
          syscalls/detail/syscall_info.h
 )
 
-if(GKFS_ENABLE_FORWARDING)
-  target_sources(
-    gkfwd_intercept
-    PUBLIC gkfs_functions.hpp
-           env.hpp
-           hooks.hpp
-           intercept.hpp
-           logging.hpp
-           make_array.hpp
-           open_file_map.hpp
-           open_dir.hpp
-           path.hpp
-           preload.hpp
-           preload_context.hpp
-           preload_util.hpp
-           rpc/rpc_types.hpp
-           rpc/forward_management.hpp
-           rpc/forward_metadata.hpp
-           rpc/forward_data.hpp
-           syscalls/args.hpp
-           syscalls/decoder.hpp
-           syscalls/errno.hpp
-           syscalls/rets.hpp
-           syscalls/syscall.hpp
-           syscalls/detail/syscall_info.h
-  )
-endif()
+target_sources(
+  gkfs_user_lib
+  PUBLIC gkfs_functions.hpp
+         env.hpp
+         hooks.hpp
+         intercept.hpp
+         logging.hpp
+         make_array.hpp
+         open_file_map.hpp
+         open_dir.hpp
+         path.hpp
+         preload.hpp
+         preload_context.hpp
+         preload_util.hpp
+         rpc/rpc_types.hpp
+         rpc/forward_management.hpp
+         rpc/forward_metadata.hpp
+         rpc/forward_data.hpp
+         syscalls/args.hpp
+         syscalls/decoder.hpp
+         syscalls/errno.hpp
+         syscalls/rets.hpp
+         syscalls/syscall.hpp
+         syscalls/detail/syscall_info.h
+         void_syscall_intercept.hpp
+)

include/client/env.hpp

@@ -48,10 +48,9 @@ static constexpr auto LOG_OUTPUT = ADD_PREFIX("LOG_OUTPUT");
 static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC");
 static constexpr auto CWD = ADD_PREFIX("CWD");
 static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE");
-#ifdef GKFS_ENABLE_FORWARDING
 static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE");
-#endif
 
+static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL");
 } // namespace gkfs::env
 
 #undef ADD_PREFIX

include/client/gkfs_functions.hpp

@@ -149,10 +149,14 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp,
 int
 gkfs_rmdir(const std::string& path);
 
+int
+gkfs_close(unsigned int fd);
+
 #ifdef HAS_RENAME
 int
 gkfs_rename(const std::string& old_path, const std::string& new_path);
 #endif // HAS_RENAME
+
 } // namespace gkfs::syscall
 
 // gkfs_getsingleserverdir is using extern "C" to demangle it for C usage

include/client/hooks.hpp

@@ -36,7 +36,13 @@ extern "C" {
 #include <sys/stat.h>
 #include <sys/syscall.h>
 }
+
+#ifndef BYPASS_SYSCALL
 #include <libsyscall_intercept_hook_point.h>
+#else
+#include <client/void_syscall_intercept.hpp>
+#endif
+
 
 /*
  * For PowerPC, syscall_no_intercept_wrapper() is defined in the

include/client/logging.hpp

@@ -30,7 +30,11 @@
 #ifndef LIBGKFS_LOGGING_HPP
 #define LIBGKFS_LOGGING_HPP
 
+#ifndef BYPASS_SYSCALL
 #include <libsyscall_intercept_hook_point.h>
+#else
+#include <client/void_syscall_intercept.hpp>
+#endif
 
 #include <type_traits>
 #include <client/make_array.hpp>

include/client/preload.hpp

@@ -31,7 +31,8 @@
 #define IOINTERCEPT_PRELOAD_HPP
 
 #include <client/preload_context.hpp>
-
+#include <common/env_util.hpp>
+#include <client/env.hpp>
 #define EUNKNOWN (-1)
 
 #define CTX gkfs::preload::PreloadContext::getInstance()
@@ -46,5 +47,10 @@ init_preload() __attribute__((constructor));
 void
 destroy_preload() __attribute__((destructor));
 
+extern "C" int
+gkfs_init();
+
+extern "C" int
+gkfs_end();
 
 #endif // IOINTERCEPT_PRELOAD_HPP

include/client/preload_context.hpp

@@ -105,6 +105,7 @@ private:
     bool internal_fds_must_relocate_;
     std::bitset<MAX_USER_FDS> protected_fds_;
     std::string hostname;
+    int replicas_;
 
 public:
     static PreloadContext*
@@ -216,6 +217,12 @@ public:
 
     std::string
     get_hostname();
+
+    void
+    set_replicas(const int repl);
+
+    int
+    get_replicas();
 };
 
 } // namespace preload

include/client/rpc/forward_data.hpp

@@ -30,6 +30,9 @@
 #ifndef GEKKOFS_CLIENT_FORWARD_DATA_HPP
 #define GEKKOFS_CLIENT_FORWARD_DATA_HPP
 
+#include <string>
+#include <memory>
+#include <set>
 namespace gkfs::rpc {
 
 struct ChunkStat {
@@ -43,14 +46,16 @@ struct ChunkStat {
 
 std::pair<int, ssize_t>
 forward_write(const std::string& path, const void* buf, off64_t offset,
-              size_t write_size);
+              size_t write_size, const int8_t num_copy = 0);
 
 std::pair<int, ssize_t>
 forward_read(const std::string& path, void* buf, off64_t offset,
-             size_t read_size);
+             size_t read_size, const int8_t num_copies,
+             std::set<int8_t>& failed);
 
 int
-forward_truncate(const std::string& path, size_t current_size, size_t new_size);
+forward_truncate(const std::string& path, size_t current_size, size_t new_size,
+                 const int8_t num_copies);
 
 std::pair<int, ChunkStat>
 forward_get_chunk_stat();

include/client/rpc/forward_metadata.hpp

@@ -50,10 +50,10 @@ class Metadata;
 namespace rpc {
 
 int
-forward_create(const std::string& path, mode_t mode);
+forward_create(const std::string& path, mode_t mode, const int copy);
 
 int
-forward_stat(const std::string& path, std::string& attr);
+forward_stat(const std::string& path, std::string& attr, const int copy);
 
 #ifdef HAS_RENAME
 int
@@ -62,22 +62,24 @@ forward_rename(const std::string& oldpath, const std::string& newpath,
 #endif // HAS_RENAME
 
 int
-forward_remove(const std::string& path);
+forward_remove(const std::string& path, const int8_t num_copies);
 
 int
-forward_decr_size(const std::string& path, size_t length);
+forward_decr_size(const std::string& path, size_t length, const int copy);
 
 int
-forward_update_metadentry(
-        const std::string& path, const gkfs::metadata::Metadata& md,
-        const gkfs::metadata::MetadentryUpdateFlags& md_flags);
+forward_update_metadentry(const std::string& path,
+                          const gkfs::metadata::Metadata& md,
+                          const gkfs::metadata::MetadentryUpdateFlags& md_flags,
+                          const int copy);
 
 std::pair<int, off64_t>
 forward_update_metadentry_size(const std::string& path, size_t size,
-                               off64_t offset, bool append_flag);
+                               off64_t offset, bool append_flag,
+                               const int num_copies);
 
 std::pair<int, off64_t>
-forward_get_metadentry_size(const std::string& path);
+forward_get_metadentry_size(const std::string& path, const int copy);
 
 std::pair<int, std::shared_ptr<gkfs::filemap::OpenDir>>
 forward_get_dirents(const std::string& path);