From 0179a9d4b1f98face46a11ca251e7692d647ce7c Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 6 Oct 2023 11:36:42 +0200 Subject: [PATCH 1/4] Solved bug in forward remap test --- tests/integration/forwarding/test_map.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/integration/forwarding/test_map.py b/tests/integration/forwarding/test_map.py index bc8e0c864..4bc63e41d 100644 --- a/tests/integration/forwarding/test_map.py +++ b/tests/integration/forwarding/test_map.py @@ -165,7 +165,7 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): lines = f.readlines() for line in lines: - if 'Forward to' in line: + if 'forwarding_mapper() Forward to' in line: ion = line.split()[-1] assert ion == '0' @@ -190,15 +190,16 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): ret = c00.write(file, buf, len(buf)) assert ret.retval == len(buf) # Return the number of read bytes - + + # log line should go to the end of the file with open(c00.log) as f: lines = f.readlines() - + ion = 0 for line in lines: - if 'Forward to' in line: + if 'forwarding_mapper() Forward to' in line: ion = line.split()[-1] - assert ion == '1' + assert ion == '1' def test_two_io_nodes_operations(gkfwd_daemon_factory, gkfwd_client_factory): """Write files from one client and read in the other using two daemons""" -- GitLab From 5814c31452f7ce0f7388a401ccb59bafb0bb4668 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 6 Oct 2023 15:23:41 +0200 Subject: [PATCH 2/4] Merging gkfs and gkfwd --- .gitlab-ci.yml | 27 +++--------- CMakeLists.txt | 6 +-- CMakePresets.json | 59 ------------------------- include/client/env.hpp | 3 +- include/client/preload.hpp | 3 +- include/daemon/classes/fs_data.hpp | 7 +++ src/client/preload.cpp | 69 +++++++++++++++--------------- src/client/preload_util.cpp | 4 -- src/daemon/classes/fs_data.cpp | 9 ++++ src/daemon/daemon.cpp | 61 +++++++++++--------------- tests/integration/harness/gkfs.py | 14 +++--- 11 files changed, 93 insertions(+), 169 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68eaca4ed..204651630 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,23 +65,6 @@ gkfs: - ${INSTALL_PATH} expire_in: 1 week -gkfwd: - stage: build - image: gekkofs/deps:0.9.2 - interruptible: true - needs: [] - script: - - cmake --preset ci-forwarding-coverage - - cmake --build ${BUILD_PATH} -j $(nproc) --target install - # reduce artifacts size - - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH} - artifacts: - paths: - - ${BUILD_PATH} - - ${INSTALL_PATH} - expire_in: 1 week - - ################################################################################ ## Testing ################################################################################ @@ -156,12 +139,12 @@ gkfwd:integration: stage: test image: gekkofs/testing:0.9.2 interruptible: true - needs: ['gkfwd'] + needs: ['gkfs'] parallel: matrix: - SUBTEST: [ forwarding ] - rules: - - when: never + # rules: + # - when: never script: ## run tests @@ -274,7 +257,7 @@ coverage:baseline: stage: report image: gekkofs/testing:0.9.2 interruptible: true - needs: ['gkfs', 'gkfwd'] + needs: ['gkfs'] script: ## capture initial coverage information to establish a baseline @@ -301,7 +284,7 @@ coverage: image: gekkofs/testing:0.9.2 # needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration', # 'gkfs:unit' ] - needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit' ] + needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] script: - cd ${CI_PROJECT_DIR} - cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index afdda9c02..adb9bb322 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -338,11 +338,7 @@ if (GKFS_BUILD_TESTS) message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}") message(STATUS "[gekkofs] Check for forwarding tests...") - if (GKFS_ENABLE_FORWARDING) - set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)") - else () - set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)") - endif () + set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)") message(STATUS "[gekkofs] Forwarding tests: ${GKFS_TESTS_FORWARDING}") message(STATUS "[gekkofs] Check for guided distributor tests...") if (GKFS_USE_GUIDED_DISTRIBUTION) diff --git a/CMakePresets.json b/CMakePresets.json index 9f12e9c73..da7bf79b3 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -27,14 +27,6 @@ "deprecated": true } }, - { - "name": "forwarding", - "inherits": "default", - "hidden": true, - "cacheVariables": { - "GKFS_ENABLE_FORWARDING": true - } - }, { "name": "release", "hidden": true, @@ -113,30 +105,6 @@ "release" ] }, - { - "name": "forwarding-debug", - "displayName": "Forwarding gekkofs (debug)", - "inherits": [ - "forwarding", - "debug" - ] - }, - { - "name": "forwarding-coverage", - "displayName": "Forwarding gekkofs (coverage)", - "inherits": [ - "forwarding", - "coverage" - ] - }, - { - "name": "forwarding-release", - "displayName": "Forwarding gekkofs (release)", - "inherits": [ - "forwarding", - "release" - ] - }, { "name": "ci-debug", "displayName": "Default gekkofs (debug, CI flags)", @@ -171,33 +139,6 @@ "default", "release" ] - }, - { - "name": "ci-forwarding-debug", - "displayName": "Forwarding gekkofs (debug, CI flags)", - "inherits": [ - "ci", - "forwarding", - "debug" - ] - }, - { - "name": "ci-forwarding-coverage", - "displayName": "Forwarding gekkofs (coverage, CI flags)", - "inherits": [ - "ci", - "forwarding", - "coverage" - ] - }, - { - "name": "ci-forwarding-release", - "displayName": "Forwarding gekkofs (release, CI flags)", - "inherits": [ - "ci", - "forwarding", - "release" - ] } ] } diff --git a/include/client/env.hpp b/include/client/env.hpp index 34a986fac..5609785e8 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -49,9 +49,8 @@ static constexpr auto LOG_PER_PROCESS = ADD_PREFIX("LOG_PER_PROCESS"); static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); static constexpr auto CWD = ADD_PREFIX("CWD"); static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); -#ifdef GKFS_ENABLE_FORWARDING static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE"); -#endif + static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL"); } // namespace gkfs::env diff --git a/include/client/preload.hpp b/include/client/preload.hpp index 175cd8f6e..6cae014b1 100644 --- a/include/client/preload.hpp +++ b/include/client/preload.hpp @@ -31,7 +31,8 @@ #define IOINTERCEPT_PRELOAD_HPP #include - +#include +#include #define EUNKNOWN (-1) #define CTX gkfs::preload::PreloadContext::getInstance() diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index 13f8023d1..ef89b5e28 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -95,6 +95,7 @@ private: bool enable_stats_ = false; bool enable_chunkstats_ = false; bool enable_prometheus_ = false; + bool enable_forwarding_ = false; std::string stats_file_; // Prometheus @@ -246,6 +247,12 @@ public: void enable_chunkstats(bool enable_chunkstats); + bool + enable_forwarding() const; + + void + enable_forwarding(bool enable_forwarding); + bool enable_prometheus() const; diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 4de0df2c3..c83333e16 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -50,13 +50,13 @@ std::unique_ptr ld_network_service; // extern variable namespace { -#ifdef GKFS_ENABLE_FORWARDING +// FORWARDING pthread_t mapper; bool forwarding_running; pthread_mutex_t remap_mutex; pthread_cond_t remap_signal; -#endif +// END FORWARDING inline void exit_error_msg(int errcode, const string& msg) { @@ -102,7 +102,6 @@ init_hermes_client() { return true; } -#ifdef GKFS_ENABLE_FORWARDING void* forwarding_mapper(void* p) { struct timespec timeout; @@ -133,18 +132,14 @@ forwarding_mapper(void* p) { return nullptr; } -#endif -#ifdef GKFS_ENABLE_FORWARDING void init_forwarding_mapper() { forwarding_running = true; pthread_create(&mapper, NULL, forwarding_mapper, NULL); } -#endif -#ifdef GKFS_ENABLE_FORWARDING void destroy_forwarding_mapper() { forwarding_running = false; @@ -153,7 +148,6 @@ destroy_forwarding_mapper() { pthread_join(mapper, NULL); } -#endif void log_prog_name() { @@ -207,31 +201,33 @@ init_environment() { } /* Setup distributor */ -#ifdef GKFS_ENABLE_FORWARDING - try { - gkfs::utils::load_forwarding_map(); + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + if(forwarding_map_file.empty()) { + try { + gkfs::utils::load_forwarding_map(); - LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); - } catch(std::exception& e) { - exit_error_msg( - EXIT_FAILURE, - fmt::format("Unable set the forwarding host '{}'", e.what())); - } + LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + } catch(std::exception& e) { + exit_error_msg(EXIT_FAILURE, + fmt::format("Unable set the forwarding host '{}'", + e.what())); + } + + auto forwarder_dist = std::make_shared( + CTX->fwd_host_id(), CTX->hosts().size()); + CTX->distributor(forwarder_dist); + } else { - auto forwarder_dist = std::make_shared( - CTX->fwd_host_id(), CTX->hosts().size()); - CTX->distributor(forwarder_dist); -#else #ifdef GKFS_USE_GUIDED_DISTRIBUTION - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); + auto distributor = std::make_shared( + CTX->local_host_id(), CTX->hosts().size()); #else - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); -#endif - CTX->distributor(distributor); + auto distributor = std::make_shared( + CTX->local_host_id(), CTX->hosts().size()); #endif - + CTX->distributor(distributor); + } LOG(INFO, "Retrieving file system configuration..."); @@ -290,9 +286,11 @@ init_preload() { CTX->unprotect_user_fds(); -#ifdef GKFS_ENABLE_FORWARDING - init_forwarding_mapper(); -#endif + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + if(forwarding_map_file.empty()) { + init_forwarding_mapper(); + } gkfs::preload::start_interception(); errno = oerrno; @@ -304,10 +302,11 @@ init_preload() { */ void destroy_preload() { -#ifdef GKFS_ENABLE_FORWARDING - destroy_forwarding_mapper(); -#endif - + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + if(forwarding_map_file.empty()) { + destroy_forwarding_mapper(); + } CTX->clear_hosts(); LOG(DEBUG, "Peer information deleted"); diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index aa09d0e07..ed9e23687 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -286,7 +286,6 @@ metadata_to_stat(const std::string& path, const gkfs::metadata::Metadata& md, return 0; } -#ifdef GKFS_ENABLE_FORWARDING map load_forwarding_map_file(const std::string& lfpath) { @@ -320,9 +319,7 @@ load_forwarding_map_file(const std::string& lfpath) { } return forwarding_map; } -#endif -#ifdef GKFS_ENABLE_FORWARDING void load_forwarding_map() { string forwarding_map_file; @@ -359,7 +356,6 @@ load_forwarding_map() { CTX->fwd_host_id(forwarding_map[local_hostname]); } -#endif vector> read_hosts_file() { diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index 0d38c718c..33b43bf59 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -256,6 +256,15 @@ FsData::enable_chunkstats(bool enable_chunkstats) { FsData::enable_chunkstats_ = enable_chunkstats; } +bool +FsData::enable_forwarding() const { + return enable_forwarding_; +} + +void +FsData::enable_forwarding(bool enable_forwarding) { + FsData::enable_forwarding_ = enable_forwarding; +} bool FsData::enable_prometheus() const { return enable_prometheus_; diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 49a87fa60..2911c6921 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -261,25 +261,6 @@ init_environment() { throw; } - GKFS_DATA->spdlogger()->debug("{}() Initializing Distributor ", __func__); - try { -#ifdef GKFS_USE_GUIDED_DISTRIBUTION - auto distributor = std::make_shared(); -#else - auto distributor = std::make_shared(); -#endif - RPC_DATA->distributor(distributor); - } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error( - "{}() Failed to initialize Distributor: {}", __func__, - e.what()); - throw; - } - -#ifdef GKFS_ENABLE_FORWARDING - GKFS_DATA->spdlogger()->debug("{}() Enable I/O forwarding mode", __func__); -#endif - #ifdef GKFS_ENABLE_AGIOS // Initialize AGIOS scheduler GKFS_DATA->spdlogger()->debug("{}() Initializing AGIOS scheduler: '{}'", @@ -552,10 +533,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) { assert(desc.count("--rootdir")); auto rootdir = opts.rootdir; -#ifdef GKFS_ENABLE_FORWARDING - // In forwarding mode, the backend is shared - auto rootdir_path = fs::path(rootdir); -#else auto rootdir_path = fs::path(rootdir); if(desc.count("--rootdir-suffix")) { if(opts.rootdir_suffix == gkfs::config::data::chunk_dir || @@ -570,7 +547,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) { rootdir_path /= opts.rootdir_suffix; GKFS_DATA->rootdir_suffix(opts.rootdir_suffix); } -#endif if(desc.count("--clean-rootdir")) { // may throw exception (caught in main) @@ -589,14 +565,23 @@ parse_input(const cli_options& opts, const CLI::App& desc) { fs::create_directories(rootdir_path); GKFS_DATA->rootdir(rootdir_path.native()); + if(desc.count("--enable-forwarding")) { + GKFS_DATA->enable_forwarding(true); + GKFS_DATA->spdlogger()->info("{}() Forwarding mode enabled", __func__); + } + if(desc.count("--metadir")) { auto metadir = opts.metadir; -#ifdef GKFS_ENABLE_FORWARDING - auto metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); -#else + auto metadir_path = fs::path(metadir); -#endif + if(GKFS_DATA->enable_forwarding()) { + // As we store normally he metadata to the pfs, we need to put each + // daemon in a separate directory. + metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); + } + + if(desc.count("--clean-rootdir")) { // may throw exception (caught in main) GKFS_DATA->spdlogger()->debug("{}() Cleaning metadir '{}' ...", @@ -613,13 +598,16 @@ parse_input(const cli_options& opts, const CLI::App& desc) { // use rootdir as metadata dir auto metadir = opts.rootdir; -#ifdef GKFS_ENABLE_FORWARDING - auto metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); - fs::create_directories(metadir_path); - GKFS_DATA->metadir(fs::canonical(metadir_path).native()); -#else - GKFS_DATA->metadir(GKFS_DATA->rootdir()); -#endif + + if(GKFS_DATA->enable_forwarding()) { + // As we store normally he metadata to the pfs, we need to put each + // daemon in a separate directory. + auto metadir_path = + fs::path(metadir) / fmt::format_int(getpid()).str(); + fs::create_directories(metadir_path); + GKFS_DATA->metadir(fs::canonical(metadir_path).native()); + } else + GKFS_DATA->metadir(GKFS_DATA->rootdir()); } if(desc.count("--dbbackend")) { @@ -787,6 +775,9 @@ main(int argc, const char* argv[]) { desc.add_option( "--output-stats", opts.stats_file, "Creates a thread that outputs the server stats each 10s to the specified file."); + desc.add_flag( + "--enable-forwarding", + "Enables forwarding mode, so the metadata is stored in a separate directory (pid)."); #ifdef GKFS_ENABLE_PROMETHEUS desc.add_flag( "--enable-prometheus", diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index ef48c4042..180946cee 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -48,15 +48,16 @@ gkfs_client_log_level = 'all' gkfs_client_log_syscall_filter = 'epoll_wait,epoll_create' gkfs_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' -gkfwd_daemon_cmd = 'gkfwd_daemon' +gkfwd_daemon_cmd = 'gkfs_daemon' gkfwd_client_cmd = 'gkfs.io' -gkfwd_client_lib_file = 'libgkfwd_intercept.so' +gkfwd_client_lib_file = 'libgkfs_intercept.so' gkfwd_hosts_file = 'gkfs_hosts.txt' gkfwd_forwarding_map_file = 'gkfs_forwarding.map' -gkfwd_daemon_log_file = 'gkfwd_daemon.log' +gkfwd_daemon_log_file = 'gkfs_daemon.log' gkfwd_daemon_log_level = '100' -gkfwd_client_log_file = 'gkfwd_client.log' +gkfwd_client_log_file = 'gkfs_client.log' gkfwd_client_log_level = 'all' +gkfwd_client_log_syscall_filter = 'epoll_wait,epoll_create' gkfwd_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' @@ -732,7 +733,8 @@ class FwdDaemon: args = [ '--mountdir', self.mountdir, '--metadir', self.metadir, '--rootdir', self.rootdir, - '-l', self._address ] + '-l', self._address, + '--enable-forwarding'] logger.debug(f"spawning daemon") logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, args))) @@ -902,7 +904,7 @@ class FwdClient: self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, - 'LD_PRELOAD' : self._preload_library, + 'LD_PRELOAD' : str(self._preload_library), 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file_local), 'LIBGKFS_LOG' : gkfs_client_log_level, -- GitLab From 899d1ca23e532042721b3451ffa68cb05c9468ac Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 9 Oct 2023 10:21:32 +0200 Subject: [PATCH 3/4] Reduced syscall log and other gkfwd fixes --- include/config.hpp | 3 ++- src/client/preload.cpp | 7 ++++--- tests/integration/harness/gkfs.py | 9 ++++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/include/config.hpp b/include/config.hpp index 7ec761f64..f74d0e8f4 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -39,7 +39,8 @@ namespace gkfs::config { constexpr auto hostfile_path = "./gkfs_hosts.txt"; -constexpr auto forwarding_file_path = "./gkfs_forwarding.map"; +// We do not default this, ENV variable always required. +constexpr auto forwarding_file_path = ""; namespace io { /* diff --git a/src/client/preload.cpp b/src/client/preload.cpp index c83333e16..5881f5e19 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -203,7 +203,8 @@ init_environment() { /* Setup distributor */ auto forwarding_map_file = gkfs::env::get_var( gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); - if(forwarding_map_file.empty()) { + + if(!forwarding_map_file.empty()) { try { gkfs::utils::load_forwarding_map(); @@ -288,7 +289,7 @@ init_preload() { auto forwarding_map_file = gkfs::env::get_var( gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); - if(forwarding_map_file.empty()) { + if(!forwarding_map_file.empty()) { init_forwarding_mapper(); } @@ -304,7 +305,7 @@ void destroy_preload() { auto forwarding_map_file = gkfs::env::get_var( gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); - if(forwarding_map_file.empty()) { + if(!forwarding_map_file.empty()) { destroy_forwarding_mapper(); } CTX->clear_hosts(); diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index 180946cee..a65a171f7 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -521,7 +521,8 @@ class ShellClient: 'LD_PRELOAD' : str(self._preload_library), 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfs_hosts_file), 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @@ -908,7 +909,8 @@ class FwdClient: 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file_local), 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file_local) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file_local), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @@ -998,7 +1000,8 @@ class ShellFwdClient: 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file), 'LIBGKFS_LOG' : gkfwd_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) -- GitLab From 9f2734bdbf41ff756a545dbf14144f9ddd420059 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Tue, 10 Oct 2023 14:35:40 +0200 Subject: [PATCH 4/4] Change CMake to remove Forward, changelog --- CHANGELOG.md | 2 + CMake/gkfs-options.cmake | 8 ---- docs/sphinx/users/dependencies.rst | 2 +- docs/sphinx/users/forwarding.rst | 12 +++--- include/CMakeLists.txt | 4 -- include/client/CMakeLists.txt | 27 -------------- include/common/CMakeLists.txt | 7 ---- include/daemon/CMakeLists.txt | 19 +--------- src/client/CMakeLists.txt | 56 +++------------------------- src/daemon/CMakeLists.txt | 60 +++--------------------------- 10 files changed, 21 insertions(+), 176 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98ae0762a..589cf59ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)). - Modified write and reads to use a bitset instead of the traditional hash per chunk in the server. - Added reattemp support in get_fs_config to other servers, when the initial server fails. +- Fused GekkoFWD and GekkoFS. GekkoFWD is enabled with the `--enable-following` in the server configuration and the ENV variable +`LIBGKFS_FORWARDING_MAP_FILE` in the clients. ([!170](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/170)). ### New ### Changed diff --git a/CMake/gkfs-options.cmake b/CMake/gkfs-options.cmake index 30da5afc3..d900cdee1 100644 --- a/CMake/gkfs-options.cmake +++ b/CMake/gkfs-options.cmake @@ -348,14 +348,6 @@ gkfs_define_option( # I/O forwarding ################################################################################ -## Forwarding support -gkfs_define_option( - GKFS_ENABLE_FORWARDING - HELP_TEXT "Enable I/O forwarding mode" - DEFAULT_VALUE OFF - DESCRIPTION "Use ${PROJECT_NAME} as an I/O forwarding layer" -) - ## Scheduling in I/O forwarding mode gkfs_define_option( GKFS_ENABLE_AGIOS diff --git a/docs/sphinx/users/dependencies.rst b/docs/sphinx/users/dependencies.rst index 0d3732410..3490ef5d6 100644 --- a/docs/sphinx/users/dependencies.rst +++ b/docs/sphinx/users/dependencies.rst @@ -43,4 +43,4 @@ Required Optional -------- -- `agios `_ (commit c26a654 or newer) to enable GekkoFWD mode. +- `agios `_ (commit c26a654 or newer) to enable GekkoFWD mode scheduling. diff --git a/docs/sphinx/users/forwarding.rst b/docs/sphinx/users/forwarding.rst index db0afc7b2..e850b4d04 100644 --- a/docs/sphinx/users/forwarding.rst +++ b/docs/sphinx/users/forwarding.rst @@ -24,13 +24,11 @@ PFS for storage, instead of a local store available at the compute nodes. Enabling GekkoFWD ------------------ -To enable the I/O forwarding mode of GekkoFS, the -:code:`GKFS_ENABLE_FORWARDING` CMake option should be enabled, :ref:`when -configuring ` the build: - -.. code-block:: console - - $ cmake -DENABLE_FORWARDING:BOOL=ON +To enable the I/O forwarding mode of GekkoFS, an environment +variable named :code:`LIBGKFS_FORWARDING_MAP_FILE` is provided +to allow users to identify the map file on each client. +This environment variable will enable GekkoFS forwarding, then we +only need to use :code:`--enable-forwarding`` option in the servers. I/O Scheduling -------------- diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index 4cc56ac38..1ae9e3b40 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -34,7 +34,3 @@ add_subdirectory(daemon) add_subdirectory(client) target_sources(gkfs_daemon PUBLIC config.hpp version.hpp.in) - -if(GKFS_ENABLE_FORWARDING) - target_sources(gkfwd_daemon PUBLIC config.hpp version.hpp.in) -endif() diff --git a/include/client/CMakeLists.txt b/include/client/CMakeLists.txt index f5df6ee5c..2d12db14e 100644 --- a/include/client/CMakeLists.txt +++ b/include/client/CMakeLists.txt @@ -54,30 +54,3 @@ target_sources( syscalls/detail/syscall_info.h ) -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_intercept - PUBLIC gkfs_functions.hpp - env.hpp - hooks.hpp - intercept.hpp - logging.hpp - make_array.hpp - open_file_map.hpp - open_dir.hpp - path.hpp - preload.hpp - preload_context.hpp - preload_util.hpp - rpc/rpc_types.hpp - rpc/forward_management.hpp - rpc/forward_metadata.hpp - rpc/forward_data.hpp - syscalls/args.hpp - syscalls/decoder.hpp - syscalls/errno.hpp - syscalls/rets.hpp - syscalls/syscall.hpp - syscalls/detail/syscall_info.h - ) -endif() diff --git a/include/common/CMakeLists.txt b/include/common/CMakeLists.txt index 956136197..38a23bc30 100644 --- a/include/common/CMakeLists.txt +++ b/include/common/CMakeLists.txt @@ -30,10 +30,3 @@ target_sources( gkfs_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp rpc/rpc_types.hpp rpc/rpc_util.hpp ) - -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp - rpc/rpc_types.hpp rpc/rpc_util.hpp - ) -endif() diff --git a/include/daemon/CMakeLists.txt b/include/daemon/CMakeLists.txt index 9cdfe06a9..26466ba16 100644 --- a/include/daemon/CMakeLists.txt +++ b/include/daemon/CMakeLists.txt @@ -38,21 +38,6 @@ target_sources( handler/rpc_util.hpp ) -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_daemon - PUBLIC daemon.hpp - util.hpp - ops/data.hpp - ops/metadentry.hpp - classes/fs_data.hpp - classes/rpc_data.hpp - handler/rpc_defs.hpp - handler/rpc_util.hpp - ) - - if(GKFS_ENABLE_AGIOS) - target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp) - endif() - +if(GKFS_ENABLE_AGIOS) + target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp) endif() diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 0108fb9c9..3055c3e92 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -52,6 +52,11 @@ target_sources( syscalls/detail/syscall_info.c ) + + if(GKFS_ENABLE_AGIOS) + target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) + endif() + target_link_libraries( gkfs_intercept PRIVATE metadata distributor env_util arithmetic path_util rpc_utils @@ -71,54 +76,3 @@ install( PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) -# ############################################################################## -# This builds the `libgkfwd_intercept.so` library: the client for GekkoFS' -# forwarding mode based on syscall interception. -# ############################################################################## - -if(GKFS_ENABLE_FORWARDING) - add_library(gkfwd_intercept SHARED) - - target_sources( - gkfwd_intercept - PRIVATE gkfs_functions.cpp - hooks.cpp - intercept.cpp - logging.cpp - open_file_map.cpp - open_dir.cpp - path.cpp - preload.cpp - preload_context.cpp - preload_util.cpp - rpc/rpc_types.cpp - rpc/forward_data.cpp - rpc/forward_management.cpp - rpc/forward_metadata.cpp - syscalls/detail/syscall_info.c - ) - target_compile_definitions(gkfwd_intercept PUBLIC GKFS_ENABLE_FORWARDING) - - if(GKFS_ENABLE_AGIOS) - target_compile_definitions(gkfwd_intercept PUBLIC GKFS_ENABLE_AGIOS) - endif() - - target_link_libraries( - gkfwd_intercept - PRIVATE metadata distributor env_util arithmetic path_util rpc_utils - PUBLIC Syscall_intercept::Syscall_intercept - dl - Mercury::Mercury - hermes - fmt::fmt - Threads::Threads - Date::TZ - ) - - install( - TARGETS gkfwd_intercept - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs - ) -endif() diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 95c4ade75..89af71c76 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -72,63 +72,15 @@ target_link_libraries( Threads::Threads ) +if(GKFS_ENABLE_AGIOS) + target_sources(gkfs_daemon PRIVATE scheduler/agios.cpp) + target_compile_definitions(gkfs_daemon PUBLIC GKFS_ENABLE_AGIOS) + target_link_libraries(gkfs_daemon PRIVATE AGIOS::AGIOS) +endif() + if(GKFS_ENABLE_CODE_COVERAGE) target_code_coverage(gkfs_daemon AUTO) endif() install(TARGETS gkfs_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -# ############################################################################## -# This builds the `gkfwd_daemon` executable: the daemon for GekkoFS data -# forwarding mode. -# ############################################################################## - -if(GKFS_ENABLE_FORWARDING) - - add_executable(gkfwd_daemon) - - # We need to add here any files that may have different compile definitions - target_sources( - gkfwd_daemon - PRIVATE ../common/rpc/rpc_util.cpp - daemon.cpp - util.cpp - ops/metadentry.cpp - ops/data.cpp - classes/fs_data.cpp - classes/rpc_data.cpp - handler/srv_metadata.cpp - handler/srv_management.cpp - handler/srv_data.cpp - ) - - target_compile_definitions(gkfwd_daemon PUBLIC GKFS_ENABLE_FORWARDING) - target_link_libraries( - gkfwd_daemon - PUBLIC # internal libs - metadata - metadata_backend - storage - distributor - statistics - log_util - env_util - path_util - # external libs - CLI11::CLI11 - fmt::fmt - Mercury::Mercury - Argobots::Argobots - Margo::Margo - # others - Threads::Threads - ) - - if(GKFS_ENABLE_AGIOS) - target_sources(gkfwd_daemon PRIVATE scheduler/agios.cpp) - target_compile_definitions(gkfwd_daemon PUBLIC GKFS_ENABLE_AGIOS) - target_link_libraries(gkfwd_daemon PRIVATE AGIOS::AGIOS) - endif() - - install(TARGETS gkfwd_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -endif () -- GitLab