From 0179a9d4b1f98face46a11ca251e7692d647ce7c Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 6 Oct 2023 11:36:42 +0200 Subject: [PATCH 01/27] Solved bug in forward remap test --- tests/integration/forwarding/test_map.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/integration/forwarding/test_map.py b/tests/integration/forwarding/test_map.py index bc8e0c864..4bc63e41d 100644 --- a/tests/integration/forwarding/test_map.py +++ b/tests/integration/forwarding/test_map.py @@ -165,7 +165,7 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): lines = f.readlines() for line in lines: - if 'Forward to' in line: + if 'forwarding_mapper() Forward to' in line: ion = line.split()[-1] assert ion == '0' @@ -190,15 +190,16 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): ret = c00.write(file, buf, len(buf)) assert ret.retval == len(buf) # Return the number of read bytes - + + # log line should go to the end of the file with open(c00.log) as f: lines = f.readlines() - + ion = 0 for line in lines: - if 'Forward to' in line: + if 'forwarding_mapper() Forward to' in line: ion = line.split()[-1] - assert ion == '1' + assert ion == '1' def test_two_io_nodes_operations(gkfwd_daemon_factory, gkfwd_client_factory): """Write files from one client and read in the other using two daemons""" -- GitLab From 5814c31452f7ce0f7388a401ccb59bafb0bb4668 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 6 Oct 2023 15:23:41 +0200 Subject: [PATCH 02/27] Merging gkfs and gkfwd --- .gitlab-ci.yml | 27 +++--------- CMakeLists.txt | 6 +-- CMakePresets.json | 59 ------------------------- include/client/env.hpp | 3 +- include/client/preload.hpp | 3 +- include/daemon/classes/fs_data.hpp | 7 +++ src/client/preload.cpp | 69 +++++++++++++++--------------- src/client/preload_util.cpp | 4 -- src/daemon/classes/fs_data.cpp | 9 ++++ src/daemon/daemon.cpp | 61 +++++++++++--------------- tests/integration/harness/gkfs.py | 14 +++--- 11 files changed, 93 insertions(+), 169 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68eaca4ed..204651630 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,23 +65,6 @@ gkfs: - ${INSTALL_PATH} expire_in: 1 week -gkfwd: - stage: build - image: gekkofs/deps:0.9.2 - interruptible: true - needs: [] - script: - - cmake --preset ci-forwarding-coverage - - cmake --build ${BUILD_PATH} -j $(nproc) --target install - # reduce artifacts size - - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH} - artifacts: - paths: - - ${BUILD_PATH} - - ${INSTALL_PATH} - expire_in: 1 week - - ################################################################################ ## Testing ################################################################################ @@ -156,12 +139,12 @@ gkfwd:integration: stage: test image: gekkofs/testing:0.9.2 interruptible: true - needs: ['gkfwd'] + needs: ['gkfs'] parallel: matrix: - SUBTEST: [ forwarding ] - rules: - - when: never + # rules: + # - when: never script: ## run tests @@ -274,7 +257,7 @@ coverage:baseline: stage: report image: gekkofs/testing:0.9.2 interruptible: true - needs: ['gkfs', 'gkfwd'] + needs: ['gkfs'] script: ## capture initial coverage information to establish a baseline @@ -301,7 +284,7 @@ coverage: image: gekkofs/testing:0.9.2 # needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration', # 'gkfs:unit' ] - needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit' ] + needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] script: - cd ${CI_PROJECT_DIR} - cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index afdda9c02..adb9bb322 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -338,11 +338,7 @@ if (GKFS_BUILD_TESTS) message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}") message(STATUS "[gekkofs] Check for forwarding tests...") - if (GKFS_ENABLE_FORWARDING) - set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)") - else () - set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)") - endif () + set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)") message(STATUS "[gekkofs] Forwarding tests: ${GKFS_TESTS_FORWARDING}") message(STATUS "[gekkofs] Check for guided distributor tests...") if (GKFS_USE_GUIDED_DISTRIBUTION) diff --git a/CMakePresets.json b/CMakePresets.json index 9f12e9c73..da7bf79b3 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -27,14 +27,6 @@ "deprecated": true } }, - { - "name": "forwarding", - "inherits": "default", - "hidden": true, - "cacheVariables": { - "GKFS_ENABLE_FORWARDING": true - } - }, { "name": "release", "hidden": true, @@ -113,30 +105,6 @@ "release" ] }, - { - "name": "forwarding-debug", - "displayName": "Forwarding gekkofs (debug)", - "inherits": [ - "forwarding", - "debug" - ] - }, - { - "name": "forwarding-coverage", - "displayName": "Forwarding gekkofs (coverage)", - "inherits": [ - "forwarding", - "coverage" - ] - }, - { - "name": "forwarding-release", - "displayName": "Forwarding gekkofs (release)", - "inherits": [ - "forwarding", - "release" - ] - }, { "name": "ci-debug", "displayName": "Default gekkofs (debug, CI flags)", @@ -171,33 +139,6 @@ "default", "release" ] - }, - { - "name": "ci-forwarding-debug", - "displayName": "Forwarding gekkofs (debug, CI flags)", - "inherits": [ - "ci", - "forwarding", - "debug" - ] - }, - { - "name": "ci-forwarding-coverage", - "displayName": "Forwarding gekkofs (coverage, CI flags)", - "inherits": [ - "ci", - "forwarding", - "coverage" - ] - }, - { - "name": "ci-forwarding-release", - "displayName": "Forwarding gekkofs (release, CI flags)", - "inherits": [ - "ci", - "forwarding", - "release" - ] } ] } diff --git a/include/client/env.hpp b/include/client/env.hpp index 34a986fac..5609785e8 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -49,9 +49,8 @@ static constexpr auto LOG_PER_PROCESS = ADD_PREFIX("LOG_PER_PROCESS"); static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); static constexpr auto CWD = ADD_PREFIX("CWD"); static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); -#ifdef GKFS_ENABLE_FORWARDING static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE"); -#endif + static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL"); } // namespace gkfs::env diff --git a/include/client/preload.hpp b/include/client/preload.hpp index 175cd8f6e..6cae014b1 100644 --- a/include/client/preload.hpp +++ b/include/client/preload.hpp @@ -31,7 +31,8 @@ #define IOINTERCEPT_PRELOAD_HPP #include - +#include +#include #define EUNKNOWN (-1) #define CTX gkfs::preload::PreloadContext::getInstance() diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index 13f8023d1..ef89b5e28 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -95,6 +95,7 @@ private: bool enable_stats_ = false; bool enable_chunkstats_ = false; bool enable_prometheus_ = false; + bool enable_forwarding_ = false; std::string stats_file_; // Prometheus @@ -246,6 +247,12 @@ public: void enable_chunkstats(bool enable_chunkstats); + bool + enable_forwarding() const; + + void + enable_forwarding(bool enable_forwarding); + bool enable_prometheus() const; diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 4de0df2c3..c83333e16 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -50,13 +50,13 @@ std::unique_ptr ld_network_service; // extern variable namespace { -#ifdef GKFS_ENABLE_FORWARDING +// FORWARDING pthread_t mapper; bool forwarding_running; pthread_mutex_t remap_mutex; pthread_cond_t remap_signal; -#endif +// END FORWARDING inline void exit_error_msg(int errcode, const string& msg) { @@ -102,7 +102,6 @@ init_hermes_client() { return true; } -#ifdef GKFS_ENABLE_FORWARDING void* forwarding_mapper(void* p) { struct timespec timeout; @@ -133,18 +132,14 @@ forwarding_mapper(void* p) { return nullptr; } -#endif -#ifdef GKFS_ENABLE_FORWARDING void init_forwarding_mapper() { forwarding_running = true; pthread_create(&mapper, NULL, forwarding_mapper, NULL); } -#endif -#ifdef GKFS_ENABLE_FORWARDING void destroy_forwarding_mapper() { forwarding_running = false; @@ -153,7 +148,6 @@ destroy_forwarding_mapper() { pthread_join(mapper, NULL); } -#endif void log_prog_name() { @@ -207,31 +201,33 @@ init_environment() { } /* Setup distributor */ -#ifdef GKFS_ENABLE_FORWARDING - try { - gkfs::utils::load_forwarding_map(); + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + if(forwarding_map_file.empty()) { + try { + gkfs::utils::load_forwarding_map(); - LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); - } catch(std::exception& e) { - exit_error_msg( - EXIT_FAILURE, - fmt::format("Unable set the forwarding host '{}'", e.what())); - } + LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + } catch(std::exception& e) { + exit_error_msg(EXIT_FAILURE, + fmt::format("Unable set the forwarding host '{}'", + e.what())); + } + + auto forwarder_dist = std::make_shared( + CTX->fwd_host_id(), CTX->hosts().size()); + CTX->distributor(forwarder_dist); + } else { - auto forwarder_dist = std::make_shared( - CTX->fwd_host_id(), CTX->hosts().size()); - CTX->distributor(forwarder_dist); -#else #ifdef GKFS_USE_GUIDED_DISTRIBUTION - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); + auto distributor = std::make_shared( + CTX->local_host_id(), CTX->hosts().size()); #else - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); -#endif - CTX->distributor(distributor); + auto distributor = std::make_shared( + CTX->local_host_id(), CTX->hosts().size()); #endif - + CTX->distributor(distributor); + } LOG(INFO, "Retrieving file system configuration..."); @@ -290,9 +286,11 @@ init_preload() { CTX->unprotect_user_fds(); -#ifdef GKFS_ENABLE_FORWARDING - init_forwarding_mapper(); -#endif + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + if(forwarding_map_file.empty()) { + init_forwarding_mapper(); + } gkfs::preload::start_interception(); errno = oerrno; @@ -304,10 +302,11 @@ init_preload() { */ void destroy_preload() { -#ifdef GKFS_ENABLE_FORWARDING - destroy_forwarding_mapper(); -#endif - + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + if(forwarding_map_file.empty()) { + destroy_forwarding_mapper(); + } CTX->clear_hosts(); LOG(DEBUG, "Peer information deleted"); diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index aa09d0e07..ed9e23687 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -286,7 +286,6 @@ metadata_to_stat(const std::string& path, const gkfs::metadata::Metadata& md, return 0; } -#ifdef GKFS_ENABLE_FORWARDING map load_forwarding_map_file(const std::string& lfpath) { @@ -320,9 +319,7 @@ load_forwarding_map_file(const std::string& lfpath) { } return forwarding_map; } -#endif -#ifdef GKFS_ENABLE_FORWARDING void load_forwarding_map() { string forwarding_map_file; @@ -359,7 +356,6 @@ load_forwarding_map() { CTX->fwd_host_id(forwarding_map[local_hostname]); } -#endif vector> read_hosts_file() { diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index 0d38c718c..33b43bf59 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -256,6 +256,15 @@ FsData::enable_chunkstats(bool enable_chunkstats) { FsData::enable_chunkstats_ = enable_chunkstats; } +bool +FsData::enable_forwarding() const { + return enable_forwarding_; +} + +void +FsData::enable_forwarding(bool enable_forwarding) { + FsData::enable_forwarding_ = enable_forwarding; +} bool FsData::enable_prometheus() const { return enable_prometheus_; diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 49a87fa60..2911c6921 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -261,25 +261,6 @@ init_environment() { throw; } - GKFS_DATA->spdlogger()->debug("{}() Initializing Distributor ", __func__); - try { -#ifdef GKFS_USE_GUIDED_DISTRIBUTION - auto distributor = std::make_shared(); -#else - auto distributor = std::make_shared(); -#endif - RPC_DATA->distributor(distributor); - } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error( - "{}() Failed to initialize Distributor: {}", __func__, - e.what()); - throw; - } - -#ifdef GKFS_ENABLE_FORWARDING - GKFS_DATA->spdlogger()->debug("{}() Enable I/O forwarding mode", __func__); -#endif - #ifdef GKFS_ENABLE_AGIOS // Initialize AGIOS scheduler GKFS_DATA->spdlogger()->debug("{}() Initializing AGIOS scheduler: '{}'", @@ -552,10 +533,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) { assert(desc.count("--rootdir")); auto rootdir = opts.rootdir; -#ifdef GKFS_ENABLE_FORWARDING - // In forwarding mode, the backend is shared - auto rootdir_path = fs::path(rootdir); -#else auto rootdir_path = fs::path(rootdir); if(desc.count("--rootdir-suffix")) { if(opts.rootdir_suffix == gkfs::config::data::chunk_dir || @@ -570,7 +547,6 @@ parse_input(const cli_options& opts, const CLI::App& desc) { rootdir_path /= opts.rootdir_suffix; GKFS_DATA->rootdir_suffix(opts.rootdir_suffix); } -#endif if(desc.count("--clean-rootdir")) { // may throw exception (caught in main) @@ -589,14 +565,23 @@ parse_input(const cli_options& opts, const CLI::App& desc) { fs::create_directories(rootdir_path); GKFS_DATA->rootdir(rootdir_path.native()); + if(desc.count("--enable-forwarding")) { + GKFS_DATA->enable_forwarding(true); + GKFS_DATA->spdlogger()->info("{}() Forwarding mode enabled", __func__); + } + if(desc.count("--metadir")) { auto metadir = opts.metadir; -#ifdef GKFS_ENABLE_FORWARDING - auto metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); -#else + auto metadir_path = fs::path(metadir); -#endif + if(GKFS_DATA->enable_forwarding()) { + // As we store normally he metadata to the pfs, we need to put each + // daemon in a separate directory. + metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); + } + + if(desc.count("--clean-rootdir")) { // may throw exception (caught in main) GKFS_DATA->spdlogger()->debug("{}() Cleaning metadir '{}' ...", @@ -613,13 +598,16 @@ parse_input(const cli_options& opts, const CLI::App& desc) { // use rootdir as metadata dir auto metadir = opts.rootdir; -#ifdef GKFS_ENABLE_FORWARDING - auto metadir_path = fs::path(metadir) / fmt::format_int(getpid()).str(); - fs::create_directories(metadir_path); - GKFS_DATA->metadir(fs::canonical(metadir_path).native()); -#else - GKFS_DATA->metadir(GKFS_DATA->rootdir()); -#endif + + if(GKFS_DATA->enable_forwarding()) { + // As we store normally he metadata to the pfs, we need to put each + // daemon in a separate directory. + auto metadir_path = + fs::path(metadir) / fmt::format_int(getpid()).str(); + fs::create_directories(metadir_path); + GKFS_DATA->metadir(fs::canonical(metadir_path).native()); + } else + GKFS_DATA->metadir(GKFS_DATA->rootdir()); } if(desc.count("--dbbackend")) { @@ -787,6 +775,9 @@ main(int argc, const char* argv[]) { desc.add_option( "--output-stats", opts.stats_file, "Creates a thread that outputs the server stats each 10s to the specified file."); + desc.add_flag( + "--enable-forwarding", + "Enables forwarding mode, so the metadata is stored in a separate directory (pid)."); #ifdef GKFS_ENABLE_PROMETHEUS desc.add_flag( "--enable-prometheus", diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index ef48c4042..180946cee 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -48,15 +48,16 @@ gkfs_client_log_level = 'all' gkfs_client_log_syscall_filter = 'epoll_wait,epoll_create' gkfs_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' -gkfwd_daemon_cmd = 'gkfwd_daemon' +gkfwd_daemon_cmd = 'gkfs_daemon' gkfwd_client_cmd = 'gkfs.io' -gkfwd_client_lib_file = 'libgkfwd_intercept.so' +gkfwd_client_lib_file = 'libgkfs_intercept.so' gkfwd_hosts_file = 'gkfs_hosts.txt' gkfwd_forwarding_map_file = 'gkfs_forwarding.map' -gkfwd_daemon_log_file = 'gkfwd_daemon.log' +gkfwd_daemon_log_file = 'gkfs_daemon.log' gkfwd_daemon_log_level = '100' -gkfwd_client_log_file = 'gkfwd_client.log' +gkfwd_client_log_file = 'gkfs_client.log' gkfwd_client_log_level = 'all' +gkfwd_client_log_syscall_filter = 'epoll_wait,epoll_create' gkfwd_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' @@ -732,7 +733,8 @@ class FwdDaemon: args = [ '--mountdir', self.mountdir, '--metadir', self.metadir, '--rootdir', self.rootdir, - '-l', self._address ] + '-l', self._address, + '--enable-forwarding'] logger.debug(f"spawning daemon") logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, args))) @@ -902,7 +904,7 @@ class FwdClient: self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, - 'LD_PRELOAD' : self._preload_library, + 'LD_PRELOAD' : str(self._preload_library), 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file_local), 'LIBGKFS_LOG' : gkfs_client_log_level, -- GitLab From 899d1ca23e532042721b3451ffa68cb05c9468ac Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 9 Oct 2023 10:21:32 +0200 Subject: [PATCH 03/27] Reduced syscall log and other gkfwd fixes --- include/config.hpp | 3 ++- src/client/preload.cpp | 7 ++++--- tests/integration/harness/gkfs.py | 9 ++++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/include/config.hpp b/include/config.hpp index 7ec761f64..f74d0e8f4 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -39,7 +39,8 @@ namespace gkfs::config { constexpr auto hostfile_path = "./gkfs_hosts.txt"; -constexpr auto forwarding_file_path = "./gkfs_forwarding.map"; +// We do not default this, ENV variable always required. +constexpr auto forwarding_file_path = ""; namespace io { /* diff --git a/src/client/preload.cpp b/src/client/preload.cpp index c83333e16..5881f5e19 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -203,7 +203,8 @@ init_environment() { /* Setup distributor */ auto forwarding_map_file = gkfs::env::get_var( gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); - if(forwarding_map_file.empty()) { + + if(!forwarding_map_file.empty()) { try { gkfs::utils::load_forwarding_map(); @@ -288,7 +289,7 @@ init_preload() { auto forwarding_map_file = gkfs::env::get_var( gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); - if(forwarding_map_file.empty()) { + if(!forwarding_map_file.empty()) { init_forwarding_mapper(); } @@ -304,7 +305,7 @@ void destroy_preload() { auto forwarding_map_file = gkfs::env::get_var( gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); - if(forwarding_map_file.empty()) { + if(!forwarding_map_file.empty()) { destroy_forwarding_mapper(); } CTX->clear_hosts(); diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index 180946cee..a65a171f7 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -521,7 +521,8 @@ class ShellClient: 'LD_PRELOAD' : str(self._preload_library), 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfs_hosts_file), 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @@ -908,7 +909,8 @@ class FwdClient: 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file_local), 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file_local) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file_local), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @@ -998,7 +1000,8 @@ class ShellFwdClient: 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file), 'LIBGKFS_LOG' : gkfwd_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file) + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file), + 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) -- GitLab From 9f2734bdbf41ff756a545dbf14144f9ddd420059 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Tue, 10 Oct 2023 14:35:40 +0200 Subject: [PATCH 04/27] Change CMake to remove Forward, changelog --- CHANGELOG.md | 2 + CMake/gkfs-options.cmake | 8 ---- docs/sphinx/users/dependencies.rst | 2 +- docs/sphinx/users/forwarding.rst | 12 +++--- include/CMakeLists.txt | 4 -- include/client/CMakeLists.txt | 27 -------------- include/common/CMakeLists.txt | 7 ---- include/daemon/CMakeLists.txt | 19 +--------- src/client/CMakeLists.txt | 56 +++------------------------- src/daemon/CMakeLists.txt | 60 +++--------------------------- 10 files changed, 21 insertions(+), 176 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98ae0762a..589cf59ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)). - Modified write and reads to use a bitset instead of the traditional hash per chunk in the server. - Added reattemp support in get_fs_config to other servers, when the initial server fails. +- Fused GekkoFWD and GekkoFS. GekkoFWD is enabled with the `--enable-following` in the server configuration and the ENV variable +`LIBGKFS_FORWARDING_MAP_FILE` in the clients. ([!170](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/170)). ### New ### Changed diff --git a/CMake/gkfs-options.cmake b/CMake/gkfs-options.cmake index 30da5afc3..d900cdee1 100644 --- a/CMake/gkfs-options.cmake +++ b/CMake/gkfs-options.cmake @@ -348,14 +348,6 @@ gkfs_define_option( # I/O forwarding ################################################################################ -## Forwarding support -gkfs_define_option( - GKFS_ENABLE_FORWARDING - HELP_TEXT "Enable I/O forwarding mode" - DEFAULT_VALUE OFF - DESCRIPTION "Use ${PROJECT_NAME} as an I/O forwarding layer" -) - ## Scheduling in I/O forwarding mode gkfs_define_option( GKFS_ENABLE_AGIOS diff --git a/docs/sphinx/users/dependencies.rst b/docs/sphinx/users/dependencies.rst index 0d3732410..3490ef5d6 100644 --- a/docs/sphinx/users/dependencies.rst +++ b/docs/sphinx/users/dependencies.rst @@ -43,4 +43,4 @@ Required Optional -------- -- `agios `_ (commit c26a654 or newer) to enable GekkoFWD mode. +- `agios `_ (commit c26a654 or newer) to enable GekkoFWD mode scheduling. diff --git a/docs/sphinx/users/forwarding.rst b/docs/sphinx/users/forwarding.rst index db0afc7b2..e850b4d04 100644 --- a/docs/sphinx/users/forwarding.rst +++ b/docs/sphinx/users/forwarding.rst @@ -24,13 +24,11 @@ PFS for storage, instead of a local store available at the compute nodes. Enabling GekkoFWD ------------------ -To enable the I/O forwarding mode of GekkoFS, the -:code:`GKFS_ENABLE_FORWARDING` CMake option should be enabled, :ref:`when -configuring ` the build: - -.. code-block:: console - - $ cmake -DENABLE_FORWARDING:BOOL=ON +To enable the I/O forwarding mode of GekkoFS, an environment +variable named :code:`LIBGKFS_FORWARDING_MAP_FILE` is provided +to allow users to identify the map file on each client. +This environment variable will enable GekkoFS forwarding, then we +only need to use :code:`--enable-forwarding`` option in the servers. I/O Scheduling -------------- diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index 4cc56ac38..1ae9e3b40 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -34,7 +34,3 @@ add_subdirectory(daemon) add_subdirectory(client) target_sources(gkfs_daemon PUBLIC config.hpp version.hpp.in) - -if(GKFS_ENABLE_FORWARDING) - target_sources(gkfwd_daemon PUBLIC config.hpp version.hpp.in) -endif() diff --git a/include/client/CMakeLists.txt b/include/client/CMakeLists.txt index f5df6ee5c..2d12db14e 100644 --- a/include/client/CMakeLists.txt +++ b/include/client/CMakeLists.txt @@ -54,30 +54,3 @@ target_sources( syscalls/detail/syscall_info.h ) -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_intercept - PUBLIC gkfs_functions.hpp - env.hpp - hooks.hpp - intercept.hpp - logging.hpp - make_array.hpp - open_file_map.hpp - open_dir.hpp - path.hpp - preload.hpp - preload_context.hpp - preload_util.hpp - rpc/rpc_types.hpp - rpc/forward_management.hpp - rpc/forward_metadata.hpp - rpc/forward_data.hpp - syscalls/args.hpp - syscalls/decoder.hpp - syscalls/errno.hpp - syscalls/rets.hpp - syscalls/syscall.hpp - syscalls/detail/syscall_info.h - ) -endif() diff --git a/include/common/CMakeLists.txt b/include/common/CMakeLists.txt index 956136197..38a23bc30 100644 --- a/include/common/CMakeLists.txt +++ b/include/common/CMakeLists.txt @@ -30,10 +30,3 @@ target_sources( gkfs_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp rpc/rpc_types.hpp rpc/rpc_util.hpp ) - -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp - rpc/rpc_types.hpp rpc/rpc_util.hpp - ) -endif() diff --git a/include/daemon/CMakeLists.txt b/include/daemon/CMakeLists.txt index 9cdfe06a9..26466ba16 100644 --- a/include/daemon/CMakeLists.txt +++ b/include/daemon/CMakeLists.txt @@ -38,21 +38,6 @@ target_sources( handler/rpc_util.hpp ) -if(GKFS_ENABLE_FORWARDING) - target_sources( - gkfwd_daemon - PUBLIC daemon.hpp - util.hpp - ops/data.hpp - ops/metadentry.hpp - classes/fs_data.hpp - classes/rpc_data.hpp - handler/rpc_defs.hpp - handler/rpc_util.hpp - ) - - if(GKFS_ENABLE_AGIOS) - target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp) - endif() - +if(GKFS_ENABLE_AGIOS) + target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp) endif() diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 0108fb9c9..3055c3e92 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -52,6 +52,11 @@ target_sources( syscalls/detail/syscall_info.c ) + + if(GKFS_ENABLE_AGIOS) + target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) + endif() + target_link_libraries( gkfs_intercept PRIVATE metadata distributor env_util arithmetic path_util rpc_utils @@ -71,54 +76,3 @@ install( PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) -# ############################################################################## -# This builds the `libgkfwd_intercept.so` library: the client for GekkoFS' -# forwarding mode based on syscall interception. -# ############################################################################## - -if(GKFS_ENABLE_FORWARDING) - add_library(gkfwd_intercept SHARED) - - target_sources( - gkfwd_intercept - PRIVATE gkfs_functions.cpp - hooks.cpp - intercept.cpp - logging.cpp - open_file_map.cpp - open_dir.cpp - path.cpp - preload.cpp - preload_context.cpp - preload_util.cpp - rpc/rpc_types.cpp - rpc/forward_data.cpp - rpc/forward_management.cpp - rpc/forward_metadata.cpp - syscalls/detail/syscall_info.c - ) - target_compile_definitions(gkfwd_intercept PUBLIC GKFS_ENABLE_FORWARDING) - - if(GKFS_ENABLE_AGIOS) - target_compile_definitions(gkfwd_intercept PUBLIC GKFS_ENABLE_AGIOS) - endif() - - target_link_libraries( - gkfwd_intercept - PRIVATE metadata distributor env_util arithmetic path_util rpc_utils - PUBLIC Syscall_intercept::Syscall_intercept - dl - Mercury::Mercury - hermes - fmt::fmt - Threads::Threads - Date::TZ - ) - - install( - TARGETS gkfwd_intercept - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs - ) -endif() diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 95c4ade75..89af71c76 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -72,63 +72,15 @@ target_link_libraries( Threads::Threads ) +if(GKFS_ENABLE_AGIOS) + target_sources(gkfs_daemon PRIVATE scheduler/agios.cpp) + target_compile_definitions(gkfs_daemon PUBLIC GKFS_ENABLE_AGIOS) + target_link_libraries(gkfs_daemon PRIVATE AGIOS::AGIOS) +endif() + if(GKFS_ENABLE_CODE_COVERAGE) target_code_coverage(gkfs_daemon AUTO) endif() install(TARGETS gkfs_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -# ############################################################################## -# This builds the `gkfwd_daemon` executable: the daemon for GekkoFS data -# forwarding mode. -# ############################################################################## - -if(GKFS_ENABLE_FORWARDING) - - add_executable(gkfwd_daemon) - - # We need to add here any files that may have different compile definitions - target_sources( - gkfwd_daemon - PRIVATE ../common/rpc/rpc_util.cpp - daemon.cpp - util.cpp - ops/metadentry.cpp - ops/data.cpp - classes/fs_data.cpp - classes/rpc_data.cpp - handler/srv_metadata.cpp - handler/srv_management.cpp - handler/srv_data.cpp - ) - - target_compile_definitions(gkfwd_daemon PUBLIC GKFS_ENABLE_FORWARDING) - target_link_libraries( - gkfwd_daemon - PUBLIC # internal libs - metadata - metadata_backend - storage - distributor - statistics - log_util - env_util - path_util - # external libs - CLI11::CLI11 - fmt::fmt - Mercury::Mercury - Argobots::Argobots - Margo::Margo - # others - Threads::Threads - ) - - if(GKFS_ENABLE_AGIOS) - target_sources(gkfwd_daemon PRIVATE scheduler/agios.cpp) - target_compile_definitions(gkfwd_daemon PUBLIC GKFS_ENABLE_AGIOS) - target_link_libraries(gkfwd_daemon PRIVATE AGIOS::AGIOS) - endif() - - install(TARGETS gkfwd_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -endif () -- GitLab From 1fe75834f64c2d976834e26cc3ef7534836ec8e0 Mon Sep 17 00:00:00 2001 From: rnou Date: Tue, 17 Oct 2023 11:20:41 +0200 Subject: [PATCH 05/27] fmt10 support --- .gitmodules | 1 + CMakeLists.txt | 2 +- include/client/logging.hpp | 58 +++++++------------ include/client/syscalls/args.hpp | 58 +++++++++---------- include/client/syscalls/decoder.hpp | 23 ++++---- include/client/syscalls/rets.hpp | 8 +-- src/client/hooks.cpp | 4 +- src/client/logging.cpp | 4 +- src/client/preload.cpp | 2 +- .../integration/harness/gkfs.io/serialize.hpp | 14 ++--- 10 files changed, 78 insertions(+), 96 deletions(-) diff --git a/.gitmodules b/.gitmodules index 18f937ae7..2d710ba8d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,7 @@ [submodule "external/hermes"] path = external/hermes url = https://github.com/bsc-ssrg/hermes.git + commit = 9f17ff97fd0b6aa44227cc164e099f25f3e5c333 [submodule "tests/scripts/bats"] path = tests/scripts/bats url = https://github.com/bats-core/bats-core.git diff --git a/CMakeLists.txt b/CMakeLists.txt index adb9bb322..cc5450476 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -225,7 +225,7 @@ include_from_source(fmt MESSAGE "[${PROJECT_NAME}] Searching for {fmt}" SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/fmt GIT_REPOSITORY https://github.com/fmtlib/fmt - GIT_TAG f94b7364b9409f05207c3af3fa4666730e11a854 # v6.1.2 + GIT_TAG e57ca2e3685b160617d3d95fcd9e789c4e06ca88 # v10.1.0 ) # ensure that fmt is linked as PIC diff --git a/include/client/logging.hpp b/include/client/logging.hpp index c99e2e4a0..dd2cc3af8 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -213,12 +213,15 @@ format_timestamp_to(Buffer&& buffer, std::chrono::microseconds{tv.tv_usec}}; if(!timezone) { - fmt::format_to(buffer, "[{}] ", now.time_since_epoch().count()); + fmt::format_to(std::back_inserter(buffer), "[{}] ", + now.time_since_epoch().count()); return; } - fmt::format_to(buffer, "[{}] ", - date::zoned_time{timezone, now}); + std::stringstream tmp; + tmp << date::zoned_time{timezone, now}; + + fmt::format_to(std::back_inserter(buffer), "[{}] ", tmp.str()); } template @@ -226,7 +229,7 @@ static inline void format_syscall_info_to(Buffer&& buffer, gkfs::syscall::info info) { const auto ttid = syscall_no_intercept(SYS_gettid); - fmt::format_to(buffer, "[{}] [syscall] ", ttid); + fmt::format_to(std::back_inserter(buffer), "[{}] [syscall] ", ttid); char o; char t; @@ -256,20 +259,15 @@ format_syscall_info_to(Buffer&& buffer, gkfs::syscall::info info) { } const std::array tmp = {'[', o, t, ']', ' '}; - fmt::format_to(buffer, fmt::string_view(tmp.data(), tmp.size())); + fmt::format_to(std::back_inserter(buffer), + fmt::string_view(tmp.data(), tmp.size())); } } // namespace detail enum { max_buffer_size = LIBGKFS_LOG_MESSAGE_SIZE }; -struct static_buffer : public fmt::basic_memory_buffer { - -protected: - void - grow(std::size_t size) override final; -}; - +using static_buffer = fmt::basic_memory_buffer; struct logger { @@ -294,15 +292,16 @@ struct logger { static_buffer buffer; detail::format_timestamp_to(buffer, timezone_); - fmt::format_to(buffer, "[{}] [{}] ", log_process_id_, - lookup_level_name(level)); + fmt::format_to(std::back_inserter(buffer), "[{}] [{}] ", , + log_process_id_, lookup_level_name(level)); if(!!(level & log::debug)) { - fmt::format_to(buffer, "<{}():{}> ", func, lineno); + fmt::format_to(std::back_inserter(buffer), "<{}():{}> ", func, + lineno); } - fmt::format_to(buffer, std::forward(args)...); - fmt::format_to(buffer, "\n"); + fmt::format_to(std::back_inserter(buffer), std::forward(args)...); + fmt::format_to(std::back_inserter(buffer), "\n"); detail::log_buffer(log_fd_, buffer); } @@ -337,8 +336,8 @@ struct logger { static_buffer prefix; detail::format_timestamp_to(prefix); - fmt::format_to(prefix, "[{}] [{}] ", log_process_id_, - lookup_level_name(level)); + fmt::format_to(std::back_inserter(prefix), "[{}] [{}] ", + log_process_id_, lookup_level_name(level)); char buffer[max_buffer_size]; const int n = vsnprintf(buffer, sizeof(buffer), fmt, ap); @@ -387,8 +386,8 @@ struct logger { } static_buffer buffer; - fmt::format_to(buffer, std::forward(args)...); - fmt::format_to(buffer, "\n"); + fmt::format_to(std::back_inserter(buffer), std::forward(args)...); + fmt::format_to(std::back_inserter(buffer), "\n"); detail::log_buffer(fd, buffer); } @@ -440,23 +439,6 @@ destroy_global_logger() { logger::global_logger().reset(); } -inline void -static_buffer::grow(std::size_t size) { - - const auto logger = get_global_logger(); - - if(logger) { - logger->log_mask_ &= ~(syscall | syscall_at_entry); - } - - std::fprintf( - stderr, - "FATAL: message too long for gkfs::log::static_buffer, increase the size of\n" - "LIBGKFS_LOG_MESSAGE_SIZE in CMake or reduce the length of the offending " - "message.\n"); - abort(); -} - } // namespace gkfs::log #define LOG(XXX, ...) LOG_##XXX(__VA_ARGS__) diff --git a/include/client/syscalls/args.hpp b/include/client/syscalls/args.hpp index 05fa82963..53ed726c3 100644 --- a/include/client/syscalls/args.hpp +++ b/include/client/syscalls/args.hpp @@ -228,7 +228,7 @@ format_flag(FmtBuffer& buffer, long flag, FlagDescriptorArray&& desc) { // we assume that if a flag value is zero, its printable // name will always be at position 0 in the array if(flag == 0 && desc[0].flag_ == 0) { - fmt::format_to(buffer, "{}", desc[0].name_); + fmt::format_to(std::back_inserter(buffer), "{}", desc[0].name_); return; } @@ -239,12 +239,12 @@ format_flag(FmtBuffer& buffer, long flag, FlagDescriptorArray&& desc) { } if((flag == desc[i].flag_)) { - fmt::format_to(buffer, "{}", desc[i].name_); + fmt::format_to(std::back_inserter(buffer), "{}", desc[i].name_); return; } } - fmt::format_to(buffer, "{:#x}", flag); + fmt::format_to(std::back_inserter(buffer), "{:#x}", flag); } template @@ -254,7 +254,7 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) { // we assume that if a flag value is zero, its printable // name will always be at position 0 in the array if(flags == 0 && desc[0].flag_ == 0) { - fmt::format_to(buffer, "{}", desc[0].name_); + fmt::format_to(std::back_inserter(buffer), "{}", desc[0].name_); return; } @@ -269,7 +269,7 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) { } if((flags & desc[i].flag_) != 0) { - fmt::format_to(buffer, "{}{}", + fmt::format_to(std::back_inserter(buffer), "{}{}", buffer.size() != buffer_start ? "|" : "", desc[i].name_); flags &= ~desc[i].flag_; @@ -280,15 +280,15 @@ format_flag_set(FmtBuffer& buffer, long flags, FlagDescriptorArray&& desc) { if(flags != 0) { if(buffer.size() != buffer_start) { - fmt::format_to(buffer, "|"); + fmt::format_to(std::back_inserter(buffer), "|"); } - fmt::format_to(buffer, "{:#x}", flags); + fmt::format_to(std::back_inserter(buffer), "{:#x}", flags); return; } if(buffer_start == buffer.size()) { - fmt::format_to(buffer, "0x0"); + fmt::format_to(std::back_inserter(buffer), "0x0"); } } @@ -311,7 +311,7 @@ format_whence_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(SEEK_END) ); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag_set(buffer, parg.value, flag_names); } @@ -334,7 +334,7 @@ format_mmap_prot_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(PROT_WRITE), FLAG_ENTRY(PROT_EXEC)); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag_set(buffer, parg.value, flag_names); return; @@ -376,7 +376,7 @@ format_mmap_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) { #endif ); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag_set(buffer, parg.value, flag_names); return; } @@ -420,14 +420,14 @@ format_clone_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(CLONE_NEWNET), FLAG_ENTRY(CLONE_IO)); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); // the low byte in clone flags contains the number of the termination // signal sent to the parent when the child dies format_flag_set(buffer, parg.value & ~0x11l, flag_names); if((parg.value & 0x11l) != 0) { - fmt::format_to(buffer, "|", parg.name); + fmt::format_to(std::back_inserter(buffer), "|", parg.name); format_signum_arg_to(buffer, {"", parg.value & 0x11l}); } return; @@ -479,7 +479,7 @@ format_signum_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(SIGSYS)); if(std::strcmp(parg.name, "")) { - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); } format_flag(buffer, parg.value, flag_names); @@ -504,7 +504,7 @@ format_sigproc_how_arg_to(FmtBuffer& buffer, const printable_arg& parg) { FLAG_ENTRY(SIG_UNBLOCK), FLAG_ENTRY(SIG_SETMASK)); - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag(buffer, parg.value, flag_names); return; } @@ -518,7 +518,7 @@ format_sigproc_how_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_none_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "void"); + fmt::format_to(std::back_inserter(buffer), "void"); } @@ -531,7 +531,7 @@ format_none_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_fd_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value)); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value)); } @@ -546,11 +546,11 @@ inline void format_atfd_arg_to(FmtBuffer& buffer, const printable_arg& parg) { if(static_cast(parg.value) == AT_FDCWD) { - fmt::format_to(buffer, "{}=AT_FDCWD", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=AT_FDCWD", parg.name); return; } - fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value)); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value)); } @@ -565,12 +565,12 @@ inline void format_cstr_arg_to(FmtBuffer& buffer, const printable_arg& parg) { if(LIKELY(reinterpret_cast(parg.value) != nullptr)) { - fmt::format_to(buffer, "{}=\"{}\"", parg.name, + fmt::format_to(std::back_inserter(buffer), "{}=\"{}\"", parg.name, reinterpret_cast(parg.value)); return; } - fmt::format_to(buffer, "{}=NULL", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=NULL", parg.name); } /** @@ -618,7 +618,7 @@ format_open_flags_to(FmtBuffer& buffer, long flags = parg.value; - fmt::format_to(buffer, "{}=", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=", parg.name); format_flag(buffer, flags & O_ACCMODE, flag_names); flags &= ~O_ACCMODE; @@ -636,7 +636,7 @@ format_open_flags_to(FmtBuffer& buffer, #endif // !O_TMPFILE if(flags != 0) { - fmt::format_to(buffer, "|", parg.name); + fmt::format_to(std::back_inserter(buffer), "|", parg.name); format_flag_set(buffer, flags, extra_flag_names); } } @@ -650,7 +650,7 @@ format_open_flags_to(FmtBuffer& buffer, template inline void format_octal_mode_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={:#04o}", parg.name, parg.value); + fmt::format_to(std::back_inserter(buffer), "{}={:#04o}", parg.name, parg.value); } /** @@ -664,12 +664,12 @@ inline void format_ptr_arg_to(FmtBuffer& buffer, const printable_arg& parg) { if(LIKELY(reinterpret_cast(parg.value) != nullptr)) { - fmt::format_to(buffer, "{}={}", parg.name, + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, reinterpret_cast(parg.value)); return; } - fmt::format_to(buffer, "{}=NULL", parg.name); + fmt::format_to(std::back_inserter(buffer), "{}=NULL", parg.name); } @@ -682,7 +682,7 @@ format_ptr_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_dec_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={}", parg.name, parg.value); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, parg.value); } @@ -695,7 +695,7 @@ format_dec_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_dec32_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={}", parg.name, static_cast(parg.value)); + fmt::format_to(std::back_inserter(buffer), "{}={}", parg.name, static_cast(parg.value)); } @@ -708,7 +708,7 @@ format_dec32_arg_to(FmtBuffer& buffer, const printable_arg& parg) { template inline void format_arg_to(FmtBuffer& buffer, const printable_arg& parg) { - fmt::format_to(buffer, "{}={:#x}", parg.name, parg.value); + fmt::format_to(std::back_inserter(buffer), "{}={:#x}", parg.name, parg.value); } #undef FLAG_ENTRY diff --git a/include/client/syscalls/decoder.hpp b/include/client/syscalls/decoder.hpp index eebdd6fdb..3172cc2df 100644 --- a/include/client/syscalls/decoder.hpp +++ b/include/client/syscalls/decoder.hpp @@ -60,19 +60,19 @@ decode(FmtBuffer& buffer, const long syscall_number, const auto sc = lookup_by_number(syscall_number, argv); - fmt::format_to(buffer, "{}(", sc.name()); + fmt::format_to(std::back_inserter(buffer), "{}(", sc.name()); for(int i = 0; i < sc.num_args(); ++i) { - const auto& arg = sc.args().at(i); + const auto arg = sc.args().at(i); arg.formatter()(buffer, {arg.name(), argv[i]}); if(i < sc.num_args() - 1) { - fmt::format_to(buffer, ", "); + fmt::format_to(std::back_inserter(buffer), ", "); } } - fmt::format_to(buffer, ") = ?"); + fmt::format_to(std::back_inserter(buffer), ") = ?"); } template @@ -84,30 +84,31 @@ decode(FmtBuffer& buffer, const long syscall_number, const long argv[MAX_ARGS], const auto sc = lookup_by_number(syscall_number, argv); - fmt::format_to(buffer, "{}(", sc.name()); + fmt::format_to(std::back_inserter(buffer), "{}(", sc.name()); for(int i = 0; i < sc.num_args(); ++i) { - const auto& arg = sc.args().at(i); + const auto arg = sc.args().at(i); arg.formatter()(buffer, {arg.name(), argv[i]}); if(i < sc.num_args() - 1) { - fmt::format_to(buffer, ", "); + fmt::format_to(std::back_inserter(buffer), ", "); } } if(never_returns(syscall_number)) { - fmt::format_to(buffer, ") = ?"); + fmt::format_to(std::back_inserter(buffer), ") = ?"); return; } if(error_code(result) != 0) { - fmt::format_to(buffer, ") = {} {} ({})", static_cast(-1), - errno_name(-result), errno_message(-result)); + fmt::format_to(std::back_inserter(buffer), ") = {} {} ({})", + static_cast(-1), errno_name(-result), + errno_message(-result)); return; } - fmt::format_to(buffer, ") = "); + fmt::format_to(std::back_inserter(buffer), ") = "); const auto& ret = sc.return_type(); ret.formatter()(buffer, result); } diff --git a/include/client/syscalls/rets.hpp b/include/client/syscalls/rets.hpp index adf77647c..1e40f77b0 100644 --- a/include/client/syscalls/rets.hpp +++ b/include/client/syscalls/rets.hpp @@ -121,7 +121,7 @@ template inline void format_none_ret_to(FmtBuffer& buffer, long val) { - fmt::format_to(buffer, "void"); + fmt::format_to(std::back_inserter(buffer), "void"); } template @@ -129,18 +129,18 @@ inline void format_ptr_ret_to(FmtBuffer& buffer, long val) { if(LIKELY(reinterpret_cast(val) != nullptr)) { - fmt::format_to(buffer, "{}", reinterpret_cast(val)); + fmt::format_to(std::back_inserter(buffer), "{}", reinterpret_cast(val)); return; } - fmt::format_to(buffer, "NULL"); + fmt::format_to(std::back_inserter(buffer), "NULL"); } template inline void format_dec_ret_to(FmtBuffer& buffer, long val) { - fmt::format_to(buffer, "{}", val); + fmt::format_to(std::back_inserter(buffer), "{}", val); } #undef LIKELY diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index 56e1f829b..a986d5898 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -878,7 +878,7 @@ hook_renameat(int olddfd, const char* oldname, int newdfd, const char* newname, "newname \"{}\", flags {}", __func__, olddfd, oldname, newdfd, newname, flags); - const char* oldpath_pass; + const char* oldpath_pass = oldname; std::string oldpath_resolved; auto oldpath_status = CTX->relativize_fd_path(olddfd, oldname, oldpath_resolved); @@ -902,7 +902,7 @@ hook_renameat(int olddfd, const char* oldname, int newdfd, const char* newname, return -EINVAL; } - const char* newpath_pass; + const char* newpath_pass = newname; std::string newpath_resolved; auto newpath_status = CTX->relativize_fd_path(newdfd, newname, newpath_resolved); diff --git a/src/client/logging.cpp b/src/client/logging.cpp index 88f994714..ee83dab22 100644 --- a/src/client/logging.cpp +++ b/src/client/logging.cpp @@ -490,7 +490,7 @@ logger::log_syscall(syscall::info info, const long syscall_number, print_syscall: - static_buffer buffer; + fmt::basic_memory_buffer buffer; detail::format_timestamp_to(buffer, timezone_); detail::format_syscall_info_to(buffer, info); @@ -501,7 +501,7 @@ print_syscall: syscall::decode(buffer, syscall_number, args); } - fmt::format_to(buffer, "\n"); + fmt::format_to(std::back_inserter(buffer), "\n"); ::syscall_no_intercept(SYS_write, log_fd_, buffer.data(), buffer.size()); } diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 5881f5e19..6d7a0c0e9 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -114,7 +114,7 @@ forwarding_mapper(void* p) { try { gkfs::utils::load_forwarding_map(); - if(previous != CTX->fwd_host_id()) { + if(previous != (int64_t) CTX->fwd_host_id()) { LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); previous = CTX->fwd_host_id(); diff --git a/tests/integration/harness/gkfs.io/serialize.hpp b/tests/integration/harness/gkfs.io/serialize.hpp index 3873b2ec8..92328fcf8 100644 --- a/tests/integration/harness/gkfs.io/serialize.hpp +++ b/tests/integration/harness/gkfs.io/serialize.hpp @@ -178,13 +178,10 @@ template <> struct adl_serializer { static void to_json(json& j, const struct ::statfs opt) { - j = json{{"f_type", opt.f_type}, - {"f_bsize", opt.f_bsize}, - {"f_blocks", opt.f_blocks}, - {"f_bfree", opt.f_bfree}, - {"f_bavail", opt.f_bavail}, - {"f_files", opt.f_files}, - {"f_ffree", opt.f_ffree}}; + j = json{{"f_type", opt.f_type}, {"f_bsize", opt.f_bsize}, + {"f_blocks", opt.f_blocks}, {"f_bfree", opt.f_bfree}, + {"f_bavail", opt.f_bavail}, {"f_files", opt.f_files}, + {"f_ffree", opt.f_ffree}}; } }; @@ -253,7 +250,8 @@ struct formatter { template auto - format(const struct ::dirent& dirent, FormatContext& ctx) { + format(const struct ::dirent& dirent, FormatContext& ctx) const + -> format_context::iterator { return format_to(ctx.out(), "struct dirent {{\n" " d_ino = {};\n" -- GitLab From 91b5aca1816b1b3bf39477aa6ba52c3c50892947 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 18 Oct 2023 13:56:30 +0200 Subject: [PATCH 06/27] Solved SegFault on date, update fmt, rocksdb 0.9.3 --- .gitlab-ci.yml | 20 ++--- CMake/FindDate.cmake | 2 +- docker/0.9.3/core/Dockerfile | 40 +++++++++ docker/0.9.3/core/Makefile | 10 +++ docker/0.9.3/deps/Dockerfile | 38 ++++++++ docker/0.9.3/deps/Makefile | 23 +++++ docker/0.9.3/docs/Dockerfile | 34 ++++++++ docker/0.9.3/docs/Makefile | 10 +++ docker/0.9.3/linter/Dockerfile | 19 ++++ docker/0.9.3/linter/Makefile | 10 +++ docker/0.9.3/testing/Dockerfile | 32 +++++++ docker/0.9.3/testing/Makefile | 10 +++ include/client/logging.hpp | 7 +- scripts/profiles/0.9.3/agios.specs | 81 +++++++++++++++++ scripts/profiles/0.9.3/all.specs | 87 +++++++++++++++++++ scripts/profiles/0.9.3/arm.specs | 83 ++++++++++++++++++ scripts/profiles/0.9.3/ci.specs | 81 +++++++++++++++++ scripts/profiles/0.9.3/default.specs | 80 +++++++++++++++++ scripts/profiles/0.9.3/direct.specs | 76 ++++++++++++++++ scripts/profiles/0.9.3/infiniband_verbs.specs | 82 +++++++++++++++++ scripts/profiles/0.9.3/install/agios.install | 57 ++++++++++++ .../profiles/0.9.3/install/argobots.install | 60 +++++++++++++ .../profiles/0.9.3/install/capstone.install | 57 ++++++++++++ scripts/profiles/0.9.3/install/curl.install | 58 +++++++++++++ scripts/profiles/0.9.3/install/date.install | 63 ++++++++++++++ scripts/profiles/0.9.3/install/json-c.install | 60 +++++++++++++ .../0.9.3/install/libfabric%verbs.install | 67 ++++++++++++++ .../profiles/0.9.3/install/libfabric.install | 67 ++++++++++++++ scripts/profiles/0.9.3/install/lz4.install | 61 +++++++++++++ scripts/profiles/0.9.3/install/margo.install | 59 +++++++++++++ .../profiles/0.9.3/install/mercury.install | 82 +++++++++++++++++ .../profiles/0.9.3/install/parallax.install | 64 ++++++++++++++ .../0.9.3/install/prometheus-cpp.install | 61 +++++++++++++ .../profiles/0.9.3/install/rocksdb.install | 79 +++++++++++++++++ .../0.9.3/install/syscall_intercept.install | 71 +++++++++++++++ scripts/profiles/0.9.3/marenostrum4.specs | 83 ++++++++++++++++++ scripts/profiles/0.9.3/mogon2.specs | 83 ++++++++++++++++++ scripts/profiles/0.9.3/ngio.specs | 83 ++++++++++++++++++ scripts/profiles/0.9.3/omnipath_psm2.specs | 82 +++++++++++++++++ scripts/profiles/0.9.3/p9.specs | 82 +++++++++++++++++ src/client/intercept.cpp | 14 +++ tests/scripts/compile_dep.sh/0.9.3/agios.out | 12 +++ tests/scripts/compile_dep.sh/0.9.3/all.out | 15 ++++ tests/scripts/compile_dep.sh/0.9.3/arm.out | 12 +++ tests/scripts/compile_dep.sh/0.9.3/ci.out | 9 ++ .../scripts/compile_dep.sh/0.9.3/default.out | 10 +++ tests/scripts/compile_dep.sh/0.9.3/direct.out | 6 ++ .../compile_dep.sh/0.9.3/infiniband_verbs.out | 10 +++ .../compile_dep.sh/0.9.3/marenostrum4.out | 13 +++ tests/scripts/compile_dep.sh/0.9.3/mogon2.out | 13 +++ tests/scripts/compile_dep.sh/0.9.3/ngio.out | 13 +++ .../compile_dep.sh/0.9.3/omnipath_psm2.out | 12 +++ tests/scripts/compile_dep.sh/0.9.3/p9.out | 12 +++ tests/scripts/dl_dep.sh/0.9.3/agios.out | 12 +++ tests/scripts/dl_dep.sh/0.9.3/all.out | 14 +++ tests/scripts/dl_dep.sh/0.9.3/arm.out | 12 +++ tests/scripts/dl_dep.sh/0.9.3/ci.out | 10 +++ tests/scripts/dl_dep.sh/0.9.3/default.out | 11 +++ tests/scripts/dl_dep.sh/0.9.3/direct.out | 7 ++ .../dl_dep.sh/0.9.3/infiniband_verbs.out | 11 +++ .../scripts/dl_dep.sh/0.9.3/marenostrum4.out | 13 +++ tests/scripts/dl_dep.sh/0.9.3/mogon2.out | 13 +++ tests/scripts/dl_dep.sh/0.9.3/ngio.out | 13 +++ .../scripts/dl_dep.sh/0.9.3/omnipath_psm2.out | 12 +++ tests/scripts/dl_dep.sh/0.9.3/p9.out | 12 +++ 65 files changed, 2480 insertions(+), 15 deletions(-) create mode 100644 docker/0.9.3/core/Dockerfile create mode 100644 docker/0.9.3/core/Makefile create mode 100644 docker/0.9.3/deps/Dockerfile create mode 100644 docker/0.9.3/deps/Makefile create mode 100644 docker/0.9.3/docs/Dockerfile create mode 100644 docker/0.9.3/docs/Makefile create mode 100644 docker/0.9.3/linter/Dockerfile create mode 100644 docker/0.9.3/linter/Makefile create mode 100644 docker/0.9.3/testing/Dockerfile create mode 100644 docker/0.9.3/testing/Makefile create mode 100644 scripts/profiles/0.9.3/agios.specs create mode 100644 scripts/profiles/0.9.3/all.specs create mode 100644 scripts/profiles/0.9.3/arm.specs create mode 100644 scripts/profiles/0.9.3/ci.specs create mode 100644 scripts/profiles/0.9.3/default.specs create mode 100644 scripts/profiles/0.9.3/direct.specs create mode 100644 scripts/profiles/0.9.3/infiniband_verbs.specs create mode 100644 scripts/profiles/0.9.3/install/agios.install create mode 100644 scripts/profiles/0.9.3/install/argobots.install create mode 100644 scripts/profiles/0.9.3/install/capstone.install create mode 100644 scripts/profiles/0.9.3/install/curl.install create mode 100644 scripts/profiles/0.9.3/install/date.install create mode 100644 scripts/profiles/0.9.3/install/json-c.install create mode 100644 scripts/profiles/0.9.3/install/libfabric%verbs.install create mode 100644 scripts/profiles/0.9.3/install/libfabric.install create mode 100644 scripts/profiles/0.9.3/install/lz4.install create mode 100644 scripts/profiles/0.9.3/install/margo.install create mode 100644 scripts/profiles/0.9.3/install/mercury.install create mode 100644 scripts/profiles/0.9.3/install/parallax.install create mode 100644 scripts/profiles/0.9.3/install/prometheus-cpp.install create mode 100644 scripts/profiles/0.9.3/install/rocksdb.install create mode 100644 scripts/profiles/0.9.3/install/syscall_intercept.install create mode 100644 scripts/profiles/0.9.3/marenostrum4.specs create mode 100644 scripts/profiles/0.9.3/mogon2.specs create mode 100644 scripts/profiles/0.9.3/ngio.specs create mode 100644 scripts/profiles/0.9.3/omnipath_psm2.specs create mode 100644 scripts/profiles/0.9.3/p9.specs create mode 100644 tests/scripts/compile_dep.sh/0.9.3/agios.out create mode 100644 tests/scripts/compile_dep.sh/0.9.3/all.out create mode 100644 tests/scripts/compile_dep.sh/0.9.3/arm.out create mode 100644 tests/scripts/compile_dep.sh/0.9.3/ci.out create mode 100644 tests/scripts/compile_dep.sh/0.9.3/default.out create mode 100644 tests/scripts/compile_dep.sh/0.9.3/direct.out create mode 100644 tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out create mode 100644 tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out create mode 100644 tests/scripts/compile_dep.sh/0.9.3/mogon2.out create mode 100644 tests/scripts/compile_dep.sh/0.9.3/ngio.out create mode 100644 tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out create mode 100644 tests/scripts/compile_dep.sh/0.9.3/p9.out create mode 100644 tests/scripts/dl_dep.sh/0.9.3/agios.out create mode 100644 tests/scripts/dl_dep.sh/0.9.3/all.out create mode 100644 tests/scripts/dl_dep.sh/0.9.3/arm.out create mode 100644 tests/scripts/dl_dep.sh/0.9.3/ci.out create mode 100644 tests/scripts/dl_dep.sh/0.9.3/default.out create mode 100644 tests/scripts/dl_dep.sh/0.9.3/direct.out create mode 100644 tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out create mode 100644 tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out create mode 100644 tests/scripts/dl_dep.sh/0.9.3/mogon2.out create mode 100644 tests/scripts/dl_dep.sh/0.9.3/ngio.out create mode 100644 tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out create mode 100644 tests/scripts/dl_dep.sh/0.9.3/p9.out diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 204651630..19c076f05 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,14 +24,14 @@ variables: GIT_SUBMODULE_STRATEGY: recursive # base image -image: gekkofs/core:0.9.2 +image: gekkofs/core:0.9.3 ################################################################################ ## Validating ################################################################################ check format: stage: lint - image: gekkofs/linter:0.9.2 + image: gekkofs/linter:0.9.3 needs: [] script: - ${SCRIPTS_DIR}/check_format.sh @@ -45,7 +45,7 @@ check format: ################################################################################ gkfs: stage: build - image: gekkofs/deps:0.9.2 + image: gekkofs/deps:0.9.3 interruptible: true needs: [] script: @@ -72,7 +72,7 @@ gkfs: ## == tests for scripts ==================== scripts: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 needs: [] script: - mkdir -p ${BUILD_PATH}/tests/scripts @@ -88,7 +88,7 @@ scripts: ## == integration tests for gkfs =========== gkfs:integration: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 interruptible: true needs: ['gkfs'] parallel: @@ -137,7 +137,7 @@ gkfs:integration: ## == integration tests for gkfwd ========== gkfwd:integration: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 interruptible: true needs: ['gkfs'] parallel: @@ -187,7 +187,7 @@ gkfwd:integration: ## == unit tests for gkfs ================== gkfs:unit: stage: test - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 needs: ['gkfs'] script: ## Add path to mkfs.kreon @@ -225,7 +225,7 @@ gkfs:unit: ################################################################################ documentation: stage: docs - image: gekkofs/docs:0.9.2 + image: gekkofs/docs:0.9.3 needs: [] rules: # we only build the documentation automatically if we are on the @@ -255,7 +255,7 @@ documentation: ## == coverage baseline ==================== coverage:baseline: stage: report - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 interruptible: true needs: ['gkfs'] @@ -281,7 +281,7 @@ coverage:baseline: coverage: stage: report - image: gekkofs/testing:0.9.2 + image: gekkofs/testing:0.9.3 # needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration', # 'gkfs:unit' ] needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] diff --git a/CMake/FindDate.cmake b/CMake/FindDate.cmake index 78b6a79d8..44d6d12ad 100644 --- a/CMake/FindDate.cmake +++ b/CMake/FindDate.cmake @@ -35,7 +35,7 @@ find_path(TZ_INCLUDE_DIR ) find_library(TZ_LIBRARY - NAMES tz + NAMES date-tz ) include(FindPackageHandleStandardArgs) diff --git a/docker/0.9.3/core/Dockerfile b/docker/0.9.3/core/Dockerfile new file mode 100644 index 000000000..a55434924 --- /dev/null +++ b/docker/0.9.3/core/Dockerfile @@ -0,0 +1,40 @@ +FROM debian:bullseye-slim + +LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + curl \ + ca-certificates \ + libtool \ + pkg-config \ + make \ + automake \ + gcc \ + g++ \ + ninja-build \ + procps \ + # AGIOS dependencies + libconfig-dev \ + # Mercury dependencies + libltdl-dev \ + lbzip2 \ + # Margo dependencies \ + libjson-c-dev \ + # RocksDB dependencies + liblz4-dev \ + # syscall_intercept dependencies + libcapstone-dev \ + # GekkoFS dependencies + libboost-program-options-dev \ + uuid-dev && \ + # install cmake 3.14+ since it's needed for some dependencies + curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \ + chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \ + ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean && \ + rm ./cmake-3.25.2-Linux-x86_64.sh diff --git a/docker/0.9.3/core/Makefile b/docker/0.9.3/core/Makefile new file mode 100644 index 000000000..9c27a583a --- /dev/null +++ b/docker/0.9.3/core/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker buildx build --platform amd64 -t gekkofs/core:0.9.3 . + +aarch64: + docker buildx build --platform aarch64 -t gekkofs/core:0.9.3 . + +all: + docker buildx build -t gekkofs/core:0.9.3 . diff --git a/docker/0.9.3/deps/Dockerfile b/docker/0.9.3/deps/Dockerfile new file mode 100644 index 000000000..ca4ecc8a6 --- /dev/null +++ b/docker/0.9.3/deps/Dockerfile @@ -0,0 +1,38 @@ +FROM gekkofs/core:0.9.3 + +LABEL Description="Debian-based environment to build GekkoFS" + +ENV GKFS_PATH /opt/gkfs +ENV GKFS_VERSION 0.9.3 + +ENV SCRIPTS_PATH ${GKFS_PATH}/scripts +ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src +ENV INSTALL_PATH /usr/local + +COPY scripts/dl_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/ +COPY scripts/patches ${SCRIPTS_PATH}/patches +COPY scripts/profiles ${SCRIPTS_PATH}/profiles + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + python3-dev \ + python3-venv \ + python3-setuptools \ + libnuma-dev libyaml-dev libcurl4-openssl-dev \ + procps && \ + python3 -m pip install --upgrade pip && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && apt-get autoclean + +# Download and build dependencies +RUN cd ${SCRIPTS_PATH} && \ + /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \ + /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \ + cp ${DEPS_SRC_PATH}/parallax/lib/include/parallax/structures.h ${INSTALL_PATH}/include/ &&\ + rm -rf ${DEPS_SRC_PATH} && \ + rm -rf ${SCRIPTS_PATH} && \ + rmdir ${GKFS_PATH} && \ + ldconfig diff --git a/docker/0.9.3/deps/Makefile b/docker/0.9.3/deps/Makefile new file mode 100644 index 000000000..a97d3195f --- /dev/null +++ b/docker/0.9.3/deps/Makefile @@ -0,0 +1,23 @@ +CWD:=$(shell pwd) +GIT_ROOT:=$(shell git rev-parse --show-toplevel) + +.PHONY: all build mount-scripts umount-scripts + +all: build mount-scripts remove-scripts +amd64: build-amd64 mount-scripts remove-scripts +aarch64: build-aarch64 mount-scripts remove-scripts + +copy-scripts: + cp -R $(GIT_ROOT)/scripts $(CWD)/scripts + +build: copy-scripts + docker build -t gekkofs/deps:0.9.3 . + +build-amd64: copy-scripts + docker build --platform amd64 -t gekkofs/deps:0.9.3 . + +build-aarch64: copy-scripts + docker build --platform aarch64 -t gekkofs/deps:0.9.3 . + +remove-scripts: + - rm -rf $(CWD)/scripts diff --git a/docker/0.9.3/docs/Dockerfile b/docker/0.9.3/docs/Dockerfile new file mode 100644 index 000000000..f770f3a23 --- /dev/null +++ b/docker/0.9.3/docs/Dockerfile @@ -0,0 +1,34 @@ +FROM gekkofs/deps:0.9.3 + +LABEL Description="Debian-based environment suitable to build GekkoFS' documentation" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # install dependencies for Doxygen + python \ + flex \ + bison \ + graphviz && \ + # install doxygen (repo version is kind of old) + cd /tmp && curl -L https://sourceforge.net/projects/doxygen/files/rel-1.9.2/doxygen-1.9.2.src.tar.gz/download --output doxygen-1.9.2.src.tar.gz && \ + tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \ + mkdir -p /tmp/doxygen-1.9.2/build && \ + cd /tmp/doxygen-1.9.2/build && \ + cmake -G "Unix Makefiles" .. && \ + make -j8 install && \ + # install sphinx, breathe and exhale + pip3 install \ + 'sphinx==4.4.0' \ + sphinx_rtd_theme \ + 'breathe==4.33.1' \ + 'exhale==0.3.1' \ + 'sphinx-copybutton==0.5.0' \ + 'sphinx-multiversion==0.2.4' \ + 'myst_parser==0.17.0' \ + attrs && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + rm -rf /tmp/doxygen-1.9.2 && \ + rm /tmp/doxygen-1.9.2.src.tar.gz && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.3/docs/Makefile b/docker/0.9.3/docs/Makefile new file mode 100644 index 000000000..c6c89926a --- /dev/null +++ b/docker/0.9.3/docs/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/docs:0.9.3 . + +aarch64: + docker build --platform aarch64 -t gekkofs/docs:0.9.3 . + +all: + docker build -t gekkofs/docs:0.9.3 . \ No newline at end of file diff --git a/docker/0.9.3/linter/Dockerfile b/docker/0.9.3/linter/Dockerfile new file mode 100644 index 000000000..f5d57118e --- /dev/null +++ b/docker/0.9.3/linter/Dockerfile @@ -0,0 +1,19 @@ +FROM gekkofs/core:0.9.3 + +LABEL Description="Debian-based environment to check the formatting of GekkoFS code" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # clang 15 deps for clang-format + lsb-release \ + wget \ + software-properties-common \ + gnupg2 && \ + # add clang-15 repos + wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 15 && \ + # install clang-format + apt-get update && apt-get install -y --no-install-recommends clang-format-15 && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.3/linter/Makefile b/docker/0.9.3/linter/Makefile new file mode 100644 index 000000000..f427eb387 --- /dev/null +++ b/docker/0.9.3/linter/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker buildx build --platform amd64 -t gekkofs/linter:0.9.3 . + +aarch64: + docker buildx build --platform aarch64 -t gekkofs/linter:0.9.3 . + +all: + docker buildx build -t gekkofs/linter:0.9.3 . diff --git a/docker/0.9.3/testing/Dockerfile b/docker/0.9.3/testing/Dockerfile new file mode 100644 index 000000000..971944a86 --- /dev/null +++ b/docker/0.9.3/testing/Dockerfile @@ -0,0 +1,32 @@ +FROM gekkofs/deps:0.9.3 + +LABEL Description="Debian-based environment to test GekkoFS" + +RUN \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + # required by lcov's genhtml + libgd-perl && \ + # install cmake 3.21+ since we need to produce JUnit XML files + curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \ + chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \ + ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \ + # install loguru + # (required by several of our scripts) + pip3 install loguru && \ + # install lcov_cobertura + # (required to produce Cobertura XML reports) + pip3 install lcov_cobertura && \ + # install lcov + # (required to produce partial coverage reports in parallel runs) + curl -OL https://github.com/linux-test-project/lcov/releases/download/v1.16/lcov-1.16.tar.gz && \ + tar xfz lcov-1.16.tar.gz && \ + cd lcov-1.16 && \ + make install && \ + cd .. && \ + # cleanup + rm -rf /var/lib/apt/lists/* && \ + apt-get clean && \ + apt-get autoclean && \ + rm ./cmake-3.25.2-Linux-x86_64.sh && \ + rm -rf ./lcov-1.16.* diff --git a/docker/0.9.3/testing/Makefile b/docker/0.9.3/testing/Makefile new file mode 100644 index 000000000..b5908b539 --- /dev/null +++ b/docker/0.9.3/testing/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/testing:0.9.3 . + +aarch64: + docker build --platform aarch64 -t gekkofs/testing:0.9.3 . + +all: + docker build -t gekkofs/testing:0.9.3 . diff --git a/include/client/logging.hpp b/include/client/logging.hpp index dd2cc3af8..464839f17 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -218,10 +218,9 @@ format_timestamp_to(Buffer&& buffer, return; } - std::stringstream tmp; - tmp << date::zoned_time{timezone, now}; - - fmt::format_to(std::back_inserter(buffer), "[{}] ", tmp.str()); + auto str = format("%F %T", date::zoned_time{timezone, now}); + + fmt::format_to(std::back_inserter(buffer), "[{}] ", str); } template diff --git a/scripts/profiles/0.9.3/agios.specs b/scripts/profiles/0.9.3/agios.specs new file mode 100644 index 000000000..0ebf47e8c --- /dev/null +++ b/scripts/profiles/0.9.3/agios.specs @@ -0,0 +1,81 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies (except transport-specific and experimental)" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.6.7" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/all.specs b/scripts/profiles/0.9.3/all.specs new file mode 100644 index 000000000..0909c9aff --- /dev/null +++ b/scripts/profiles/0.9.3/all.specs @@ -0,0 +1,87 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.6.7" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" + ["curl"]="7.82.0" + ["prometheus-cpp"]="v1.0.0" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "agios" "curl" "prometheus-cpp" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/arm.specs b/scripts/profiles/0.9.3/arm.specs new file mode 100644 index 000000000..395c43610 --- /dev/null +++ b/scripts/profiles/0.9.3/arm.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for PowerPC supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.6.7" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="fb73c3924b502e2840a5dc8a18746e395b06a8a6" + ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch.arm64" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" + ["syscall_intercept"]="ARM64" +) diff --git a/scripts/profiles/0.9.3/ci.specs b/scripts/profiles/0.9.3/ci.specs new file mode 100644 index 000000000..2f2f22f10 --- /dev/null +++ b/scripts/profiles/0.9.3/ci.specs @@ -0,0 +1,81 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies required by the CI" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["argobots"]="1.1" + ["rocksdb"]="8.6.7" + ["prometheus-cpp"]="v1.0.0" + ) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" + ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" + ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" + ) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "date" "agios" "parallax" "prometheus-cpp" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/default.specs b/scripts/profiles/0.9.3/default.specs new file mode 100644 index 000000000..821c93b7b --- /dev/null +++ b/scripts/profiles/0.9.3/default.specs @@ -0,0 +1,80 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="All dependencies" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.6.7" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch syscall_intercept_clone3.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/direct.specs b/scripts/profiles/0.9.3/direct.specs new file mode 100644 index 000000000..1410254ba --- /dev/null +++ b/scripts/profiles/0.9.3/direct.specs @@ -0,0 +1,76 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Direct dependencies" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["argobots"]="1.1" + ["rocksdb"]="8.6.7" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( +) diff --git a/scripts/profiles/0.9.3/infiniband_verbs.specs b/scripts/profiles/0.9.3/infiniband_verbs.specs new file mode 100644 index 000000000..a57dd217e --- /dev/null +++ b/scripts/profiles/0.9.3/infiniband_verbs.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Infiniband supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.6.7" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric%verbs"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading/installing +order=( + "lz4" "capstone" "json-c" "libfabric%verbs" "mercury" "argobots" "margo" "rocksdb" + "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric%verbs"]="--enable-verbs=yes" +) diff --git a/scripts/profiles/0.9.3/install/agios.install b/scripts/profiles/0.9.3/install/agios.install new file mode 100644 index 000000000..3d0392ff5 --- /dev/null +++ b/scripts/profiles/0.9.3/install/agios.install @@ -0,0 +1,57 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="agios" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" .. + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/argobots.install b/scripts/profiles/0.9.3/install/argobots.install new file mode 100644 index 000000000..96ccea920 --- /dev/null +++ b/scripts/profiles/0.9.3/install/argobots.install @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="argobots" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" --enable-perf-opt --disable-checks + make -j"${CORES}" + make install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/capstone.install b/scripts/profiles/0.9.3/install/capstone.install new file mode 100644 index 000000000..ff299d82b --- /dev/null +++ b/scripts/profiles/0.9.3/install/capstone.install @@ -0,0 +1,57 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="capstone" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/curl.install b/scripts/profiles/0.9.3/install/curl.install new file mode 100644 index 000000000..d873819c4 --- /dev/null +++ b/scripts/profiles/0.9.3/install/curl.install @@ -0,0 +1,58 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="curl" + CURR="${SOURCE_DIR}/${ID}" + cd "${CURR}" + autoreconf -fi + ./configure --prefix="${INSTALL_DIR}" --without-ssl + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/date.install b/scripts/profiles/0.9.3/install/date.install new file mode 100644 index 000000000..7e5dd6b31 --- /dev/null +++ b/scripts/profiles/0.9.3/install/date.install @@ -0,0 +1,63 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="date" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_CXX_STANDARD:STRING=14 \ + -DBUILD_TZ_LIB:BOOL=ON \ + -DUSE_SYSTEM_TZ_DB:BOOL=ON \ + -DBUILD_SHARED_LIBS:BOOL=ON .. + + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/json-c.install b/scripts/profiles/0.9.3/install/json-c.install new file mode 100644 index 000000000..7280cda81 --- /dev/null +++ b/scripts/profiles/0.9.3/install/json-c.install @@ -0,0 +1,60 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CURR="${SOURCE_DIR}/json-c" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + cmake -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" -DCMAKE_BUILD_TYPE:STRING=Release .. + make -j"${CORES}" install + # Margo doesn't search in both directories, so we make it available in both lib and lib64 + if [[ -f "${INSTALL_DIR}/lib64/pkgconfig/json-c.pc" ]]; then + cp ${INSTALL_DIR}/lib64/pkgconfig/json-c.pc ${INSTALL_DIR}/lib/pkgconfig/ + fi +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/libfabric%verbs.install b/scripts/profiles/0.9.3/install/libfabric%verbs.install new file mode 100644 index 000000000..15257eb81 --- /dev/null +++ b/scripts/profiles/0.9.3/install/libfabric%verbs.install @@ -0,0 +1,67 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="libfabric%verbs" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/libfabric.install b/scripts/profiles/0.9.3/install/libfabric.install new file mode 100644 index 000000000..7db89438e --- /dev/null +++ b/scripts/profiles/0.9.3/install/libfabric.install @@ -0,0 +1,67 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID=libfabric + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + OFI_CONFIG="../configure --prefix=${INSTALL_DIR} --enable-tcp=yes" + + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + + if [[ -n "${EXTRA_INSTALL_ARGS}" ]]; then + OFI_CONFIG="${OFI_CONFIG} ${EXTRA_INSTALL_ARGS}" + fi + + ${OFI_CONFIG} + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/lz4.install b/scripts/profiles/0.9.3/install/lz4.install new file mode 100644 index 000000000..13e4e53b3 --- /dev/null +++ b/scripts/profiles/0.9.3/install/lz4.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="lz4" + CURR="${SOURCE_DIR}/${ID}" + cd "${CURR}" + # try to remove binaries first in case they already exist. Otherwise install fails. + LZ4_BINS=("${INSTALL_DIR}"/bin/lz4c "${INSTALL_DIR}"/bin/lz4cat "${INSTALL_DIR}"/bin/unlz4 "${INSTALL_DIR}"/bin/lz4) + for LZ4_BIN in "${LZ4_BINS[@]}"; do + [ -e "$LZ4_BIN" ] && rm "$LZ4_BIN" + done + make -j"${CORES}" + make DESTDIR="${INSTALL_DIR}" PREFIX="" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/margo.install b/scripts/profiles/0.9.3/install/margo.install new file mode 100644 index 000000000..d2c6e4b5f --- /dev/null +++ b/scripts/profiles/0.9.3/install/margo.install @@ -0,0 +1,59 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="margo" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./prepare.sh + cd "${CURR}/build" + ../configure --prefix="${INSTALL_DIR}" PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" CFLAGS="${CFLAGS} -Wall -O3" + make -j"${CORES}" install +} + +pkg_check() { + make check +} diff --git a/scripts/profiles/0.9.3/install/mercury.install b/scripts/profiles/0.9.3/install/mercury.install new file mode 100644 index 000000000..601305b83 --- /dev/null +++ b/scripts/profiles/0.9.3/install/mercury.install @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + # if the profile compiles bmi, enable it + if [[ -n "${PROFILE_DEP_NAMES['bmi']}" ]]; then + USE_BMI="-DNA_USE_BMI:BOOL=ON" + else + USE_BMI="-DNA_USE_BMI:BOOL=OFF" + fi + + # if the profile provides any flavour of libfabric, enable it + if profile_has_dependency "^libfabric.*$"; then + USE_OFI="-DNA_USE_OFI:BOOL=ON" + else + USE_OFI="-DNA_USE_OFI:BOOL=OFF" + fi + + ID="mercury" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_PREFIX_PATH=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DBUILD_TESTING:BOOL=ON \ + -DMERCURY_USE_CHECKSUMS:BOOL=OFF \ + -DMERCURY_USE_BOOST_PP:BOOL=ON \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + ${USE_BMI} ${USE_OFI} \ + .. + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/parallax.install b/scripts/profiles/0.9.3/install/parallax.install new file mode 100644 index 000000000..38bd9c689 --- /dev/null +++ b/scripts/profiles/0.9.3/install/parallax.install @@ -0,0 +1,64 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + + CURR="${SOURCE_DIR}/parallax" + # sed -i -e 's/KEY_SIZE (256)/KEY_SIZE (4096)/g' ${CURR}/lib/btree/conf.h + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_CXX_FLAGS_RELEASE="-Wno-error=unused-result" \ + -DDISABLE_LOGGING:BOOL=ON \ + .. + make -j"${CORES}" + make install + # We need to copy this file as it is not installed + cp ${CURR}/lib/include/parallax/structures.h ${INSTALL_DIR}/include/ +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/prometheus-cpp.install b/scripts/profiles/0.9.3/install/prometheus-cpp.install new file mode 100644 index 000000000..62e420217 --- /dev/null +++ b/scripts/profiles/0.9.3/install/prometheus-cpp.install @@ -0,0 +1,61 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="prometheus-cpp" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + ${CMAKE} \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DBUILD_SHARED_LIBS:BOOL=ON \ + .. + make -j"${CORES}" install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/rocksdb.install b/scripts/profiles/0.9.3/install/rocksdb.install new file mode 100644 index 000000000..6ece050bd --- /dev/null +++ b/scripts/profiles/0.9.3/install/rocksdb.install @@ -0,0 +1,79 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + CXXFLAGS='' + # gcc 9 and clang 8 need -Wno-error=deprecated-copy -Wno-error=pessimizing-move + if [[ ("${COMPILER_NAME}" == "g++" && "${COMPILER_MAJOR_VERSION}" -ge 9) || + ("${COMPILER_NAME}" == "clang" && "${COMPILER_MAJOR_VERSION}" -ge 8) ]]; then + CXXFLAGS='-Wno-error=deprecated-copy -Wno-error=pessimizing-move -Wno-error=maybe-uninitialized' + fi + + # TODO use SSE? + CURR="${SOURCE_DIR}/rocksdb" + prepare_build_dir "${CURR}" + cd "${CURR}/build" + PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig" $CMAKE \ + -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR="${INSTALL_DIR}/lib" \ + -DCMAKE_INSTALL_INCLUDEDIR="${INSTALL_DIR}/include" \ + -DROCKSDB_BUILD_SHARED=OFF \ + -DWITH_LZ4=ON \ + -DWITH_GFLAGS=OFF \ + -DUSE_RTTI=1 \ + -DPORTABLE=1 \ + -DWITH_ALL_TESTS=OFF \ + -DWITH_BENCHMARK_TOOLS=OFF \ + -DWITH_TOOLS=OFF .. + make -j"${CORES}" install + + +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/install/syscall_intercept.install b/scripts/profiles/0.9.3/install/syscall_intercept.install new file mode 100644 index 000000000..5c862dcb7 --- /dev/null +++ b/scripts/profiles/0.9.3/install/syscall_intercept.install @@ -0,0 +1,71 @@ +################################################################################ +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ +# vi: ft=bash + +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ + +pkg_install() { + ID="syscall_intercept" + CURR="${SOURCE_DIR}/${ID}" + EXTRA_INSTALL_ARGS="${PROFILE_EXTRA_INSTALL_ARGS[${ID}]}" + prepare_build_dir "${CURR}" + if [[ ${EXTRA_INSTALL_ARGS} == "ARM64" ]]; then + cd "${CURR}"/arch/aarch64/ + mkdir -p build + cd build + else + cd "${CURR}"/build + fi + $CMAKE -DCMAKE_PREFIX_PATH="${INSTALL_DIR}" \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ + -DCMAKE_BUILD_TYPE:STRING=Debug \ + -DBUILD_EXAMPLES:BOOL=OFF \ + -DBUILD_TESTS:BOOK=OFF .. + make -j"${CORES}" install + if [[ ${EXTRA_INSTALL_ARGS} == "ARM64" ]]; then + cp "${CURR}"/arch/aarch64/include/libsyscall_intercept_hook_point.h ${INSTALL_DIR}/include + fi +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/marenostrum4.specs b/scripts/profiles/0.9.3/marenostrum4.specs new file mode 100644 index 000000000..20598044d --- /dev/null +++ b/scripts/profiles/0.9.3/marenostrum4.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Marenostrum 4 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.6.7" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" +) diff --git a/scripts/profiles/0.9.3/mogon2.specs b/scripts/profiles/0.9.3/mogon2.specs new file mode 100644 index 000000000..3b33fbad4 --- /dev/null +++ b/scripts/profiles/0.9.3/mogon2.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Mogon 2 supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.6.7" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/ngio.specs b/scripts/profiles/0.9.3/ngio.specs new file mode 100644 index 000000000..7a73e8e40 --- /dev/null +++ b/scripts/profiles/0.9.3/ngio.specs @@ -0,0 +1,83 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for NEXTGenIO prototype cluster" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.6.7" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" + ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/omnipath_psm2.specs b/scripts/profiles/0.9.3/omnipath_psm2.specs new file mode 100644 index 000000000..66f62bb10 --- /dev/null +++ b/scripts/profiles/0.9.3/omnipath_psm2.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for Omnipath supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.6.7" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" +) diff --git a/scripts/profiles/0.9.3/p9.specs b/scripts/profiles/0.9.3/p9.specs new file mode 100644 index 000000000..efa069ebc --- /dev/null +++ b/scripts/profiles/0.9.3/p9.specs @@ -0,0 +1,82 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# vi: ft=bash + +# Variables to be imported into the scripts +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order + +# Comment that should be displayed when printing the profile +comment="Dependencies for PowerPC supercomputer" + +# Dependencies that must be downloaded directly +wgetdeps=( + ["lz4"]="1.9.3" + ["capstone"]="4.0.2" + ["argobots"]="1.1" + ["rocksdb"]="8.6.7" + ["psm2"]="11.2.185" + ["json-c"]="0.15-20200726" +) + +# Dependencies that must be cloned +clonedeps=( + ["libfabric"]="HEAD@v1.13.2" + ["mercury"]="v2.1.0" + ["margo"]="v0.9.6" + ["syscall_intercept"]="6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f" + ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" +) + +# Extra arguments for git clone +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) + +# Patches that should be applied post-clone +clonedeps_patches=( + ["syscall_intercept"]="syscall_intercept.patch" +) + +# Ordering that MUST be followed when downloading +order=( + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" +) + +# Extra arguments passed to the installation script. As such, they can +# reference the following variables: +# - CMAKE: a variable that expands to the cmake binary +# - SOURCE_DIR: the directory where the sources for the package were +# downloaded +# - INSTALL_DIR: the directory where the package should be installed +# - CORES: the number of cores to use when building +# - PERFORM_TEST: whether tests for the package should be executed +extra_install_args=( + ["libfabric"]="--enable-psm2=no --enable-sockets=yes" +) diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index 316c5e398..77a277f5e 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -84,6 +84,11 @@ hook_internal(long syscall_number, long arg0, long arg1, long arg2, long arg3, arg3, arg4, arg5}; #endif + // We avoid logging sched_getaffinity to use tz.h + if(syscall_number == SYS_sched_getaffinity) { + return gkfs::syscall::forward_to_kernel; + } + LOG(SYSCALL, gkfs::syscall::from_internal_code | gkfs::syscall::to_hook | gkfs::syscall::not_executed, @@ -423,6 +428,12 @@ hook(long syscall_number, long arg0, long arg1, long arg2, long arg3, long arg4, arg3, arg4, arg5}; #endif + // We avoid logging sched_getaffinity to use tz.h + if(syscall_number == SYS_sched_getaffinity) { + return gkfs::syscall::forward_to_kernel; + } + + LOG(SYSCALL, gkfs::syscall::from_external_code | gkfs::syscall::to_hook | gkfs::syscall::not_executed, @@ -845,6 +856,9 @@ hook_forwarded_syscall(long syscall_number, long arg0, long arg1, long arg2, const long args[gkfs::syscall::MAX_ARGS] = {arg0, arg1, arg2, arg3, arg4, arg5}; #endif + if(syscall_number == SYS_sched_getaffinity) { + return; + } LOG(SYSCALL, ::get_current_syscall_info() | gkfs::syscall::executed, syscall_number, args, result); diff --git a/tests/scripts/compile_dep.sh/0.9.3/agios.out b/tests/scripts/compile_dep.sh/0.9.3/agios.out new file mode 100644 index 000000000..b098aecfa --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/agios.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### + diff --git a/tests/scripts/compile_dep.sh/0.9.3/all.out b/tests/scripts/compile_dep.sh/0.9.3/all.out new file mode 100644 index 000000000..5cdf4c5ff --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/all.out @@ -0,0 +1,15 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### +Done \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.3/arm.out b/tests/scripts/compile_dep.sh/0.9.3/arm.out new file mode 100644 index 000000000..7138da9e2 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/arm.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/ci.out b/tests/scripts/compile_dep.sh/0.9.3/ci.out new file mode 100644 index 000000000..d3c276800 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/ci.out @@ -0,0 +1,9 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: agios ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/default.out b/tests/scripts/compile_dep.sh/0.9.3/default.out new file mode 100644 index 000000000..787e4a08f --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/default.out @@ -0,0 +1,10 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/direct.out b/tests/scripts/compile_dep.sh/0.9.3/direct.out new file mode 100644 index 000000000..816266773 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/direct.out @@ -0,0 +1,6 @@ +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out b/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out new file mode 100644 index 000000000..e82dd020f --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out @@ -0,0 +1,10 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: libfabric%verbs ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out b/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out new file mode 100644 index 000000000..38f67ce3d --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out @@ -0,0 +1,13 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/mogon2.out b/tests/scripts/compile_dep.sh/0.9.3/mogon2.out new file mode 100644 index 000000000..38f67ce3d --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/mogon2.out @@ -0,0 +1,13 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/ngio.out b/tests/scripts/compile_dep.sh/0.9.3/ngio.out new file mode 100644 index 000000000..38f67ce3d --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/ngio.out @@ -0,0 +1,13 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### +######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out b/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out new file mode 100644 index 000000000..7138da9e2 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/p9.out b/tests/scripts/compile_dep.sh/0.9.3/p9.out new file mode 100644 index 000000000..7138da9e2 --- /dev/null +++ b/tests/scripts/compile_dep.sh/0.9.3/p9.out @@ -0,0 +1,12 @@ +######## Installing: lz4 ############################### +######## Installing: capstone ############################### +######## Installing: json-c ############################### +######## Installing: psm2 ############################### +WARNING: Install script for 'psm2' not found. Skipping. +######## Installing: libfabric ############################### +######## Installing: mercury ############################### +######## Installing: argobots ############################### +######## Installing: margo ############################### +######## Installing: rocksdb ############################### +######## Installing: syscall_intercept ############################### +######## Installing: date ############################### diff --git a/tests/scripts/dl_dep.sh/0.9.3/agios.out b/tests/scripts/dl_dep.sh/0.9.3/agios.out new file mode 100644 index 000000000..1606211fa --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/agios.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/all.out b/tests/scripts/dl_dep.sh/0.9.3/all.out new file mode 100644 index 000000000..4a1ab9e6f --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/all.out @@ -0,0 +1,14 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/arm.out b/tests/scripts/dl_dep.sh/0.9.3/arm.out new file mode 100644 index 000000000..2c61edb9c --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/arm.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[fb73c3924b502e2840a5dc8a18746e395b06a8a6]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Done diff --git a/tests/scripts/dl_dep.sh/0.9.3/ci.out b/tests/scripts/dl_dep.sh/0.9.3/ci.out new file mode 100644 index 000000000..4ee089ef7 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/ci.out @@ -0,0 +1,10 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/default.out b/tests/scripts/dl_dep.sh/0.9.3/default.out new file mode 100644 index 000000000..a68701cf0 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/default.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/direct.out b/tests/scripts/dl_dep.sh/0.9.3/direct.out new file mode 100644 index 000000000..e27eae530 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/direct.out @@ -0,0 +1,7 @@ +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out b/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out new file mode 100644 index 000000000..6ebc289ff --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out @@ -0,0 +1,11 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out new file mode 100644 index 000000000..657154fdc --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/mogon2.out b/tests/scripts/dl_dep.sh/0.9.3/mogon2.out new file mode 100644 index 000000000..657154fdc --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/mogon2.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/ngio.out b/tests/scripts/dl_dep.sh/0.9.3/ngio.out new file mode 100644 index 000000000..657154fdc --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/ngio.out @@ -0,0 +1,13 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out new file mode 100644 index 000000000..d26b498ca --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/p9.out b/tests/scripts/dl_dep.sh/0.9.3/p9.out new file mode 100644 index 000000000..3a99bb523 --- /dev/null +++ b/tests/scripts/dl_dep.sh/0.9.3/p9.out @@ -0,0 +1,12 @@ +Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' +Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' +Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' +Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' +Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' +Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' +Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' +Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f]' and flags '' +Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' +Done -- GitLab From 23a6257a655237df426bab2498a40371f293b204 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 18 Oct 2023 14:30:05 +0200 Subject: [PATCH 07/27] Format logging --- include/client/logging.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/client/logging.hpp b/include/client/logging.hpp index 464839f17..3224f9ce2 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -218,8 +218,10 @@ format_timestamp_to(Buffer&& buffer, return; } - auto str = format("%F %T", date::zoned_time{timezone, now}); - + auto str = + format("%F %T", + date::zoned_time{timezone, now}); + fmt::format_to(std::back_inserter(buffer), "[{}] ", str); } -- GitLab From 58ba893a86370f0ccaa32e9236a3efe8b52ebb3c Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 18 Oct 2023 20:14:31 +0200 Subject: [PATCH 08/27] Avoid reentrance on logging --- src/client/intercept.cpp | 46 ++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index 77a277f5e..a58383d78 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -48,7 +48,7 @@ namespace { thread_local bool reentrance_guard_flag; thread_local gkfs::syscall::info saved_syscall_info; - +thread_local bool avoid_logging; constexpr void save_current_syscall_info(gkfs::syscall::info info) { saved_syscall_info = info; @@ -84,16 +84,15 @@ hook_internal(long syscall_number, long arg0, long arg1, long arg2, long arg3, arg3, arg4, arg5}; #endif - // We avoid logging sched_getaffinity to use tz.h - if(syscall_number == SYS_sched_getaffinity) { - return gkfs::syscall::forward_to_kernel; + if(!avoid_logging) { + avoid_logging = true; + LOG(SYSCALL, + gkfs::syscall::from_internal_code | gkfs::syscall::to_hook | + gkfs::syscall::not_executed, + syscall_number, args); + avoid_logging = false; } - LOG(SYSCALL, - gkfs::syscall::from_internal_code | gkfs::syscall::to_hook | - gkfs::syscall::not_executed, - syscall_number, args); - switch(syscall_number) { #ifdef SYS_open case SYS_open: @@ -428,17 +427,14 @@ hook(long syscall_number, long arg0, long arg1, long arg2, long arg3, long arg4, arg3, arg4, arg5}; #endif - // We avoid logging sched_getaffinity to use tz.h - if(syscall_number == SYS_sched_getaffinity) { - return gkfs::syscall::forward_to_kernel; + if(!avoid_logging) { + avoid_logging = true; + LOG(SYSCALL, + gkfs::syscall::from_external_code | gkfs::syscall::to_hook | + gkfs::syscall::not_executed, + syscall_number, args); + avoid_logging = false; } - - - LOG(SYSCALL, - gkfs::syscall::from_external_code | gkfs::syscall::to_hook | - gkfs::syscall::not_executed, - syscall_number, args); - switch(syscall_number) { case SYS_execve: @@ -856,13 +852,13 @@ hook_forwarded_syscall(long syscall_number, long arg0, long arg1, long arg2, const long args[gkfs::syscall::MAX_ARGS] = {arg0, arg1, arg2, arg3, arg4, arg5}; #endif - if(syscall_number == SYS_sched_getaffinity) { - return; - } - - LOG(SYSCALL, ::get_current_syscall_info() | gkfs::syscall::executed, - syscall_number, args, result); + if(!avoid_logging) { + avoid_logging = true; + LOG(SYSCALL, ::get_current_syscall_info() | gkfs::syscall::executed, + syscall_number, args, result); + avoid_logging = false; + } ::reset_current_syscall_info(); } -- GitLab From 6383546c6f53c68435faab9a7dcc7d0a6ac43ec8 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 19 Oct 2023 12:58:00 +0200 Subject: [PATCH 09/27] thread_local to avoid log reentrance --- src/client/intercept.cpp | 36 +++++++++++++----------------------- src/client/logging.cpp | 8 ++++++++ 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index a58383d78..316c5e398 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -48,7 +48,7 @@ namespace { thread_local bool reentrance_guard_flag; thread_local gkfs::syscall::info saved_syscall_info; -thread_local bool avoid_logging; + constexpr void save_current_syscall_info(gkfs::syscall::info info) { saved_syscall_info = info; @@ -84,14 +84,10 @@ hook_internal(long syscall_number, long arg0, long arg1, long arg2, long arg3, arg3, arg4, arg5}; #endif - if(!avoid_logging) { - avoid_logging = true; - LOG(SYSCALL, - gkfs::syscall::from_internal_code | gkfs::syscall::to_hook | - gkfs::syscall::not_executed, - syscall_number, args); - avoid_logging = false; - } + LOG(SYSCALL, + gkfs::syscall::from_internal_code | gkfs::syscall::to_hook | + gkfs::syscall::not_executed, + syscall_number, args); switch(syscall_number) { #ifdef SYS_open @@ -427,14 +423,11 @@ hook(long syscall_number, long arg0, long arg1, long arg2, long arg3, long arg4, arg3, arg4, arg5}; #endif - if(!avoid_logging) { - avoid_logging = true; - LOG(SYSCALL, - gkfs::syscall::from_external_code | gkfs::syscall::to_hook | - gkfs::syscall::not_executed, - syscall_number, args); - avoid_logging = false; - } + LOG(SYSCALL, + gkfs::syscall::from_external_code | gkfs::syscall::to_hook | + gkfs::syscall::not_executed, + syscall_number, args); + switch(syscall_number) { case SYS_execve: @@ -853,12 +846,9 @@ hook_forwarded_syscall(long syscall_number, long arg0, long arg1, long arg2, arg3, arg4, arg5}; #endif - if(!avoid_logging) { - avoid_logging = true; - LOG(SYSCALL, ::get_current_syscall_info() | gkfs::syscall::executed, - syscall_number, args, result); - avoid_logging = false; - } + LOG(SYSCALL, ::get_current_syscall_info() | gkfs::syscall::executed, + syscall_number, args, result); + ::reset_current_syscall_info(); } diff --git a/src/client/logging.cpp b/src/client/logging.cpp index ee83dab22..e8a243e45 100644 --- a/src/client/logging.cpp +++ b/src/client/logging.cpp @@ -46,6 +46,9 @@ extern "C" { namespace fs = std::filesystem; +namespace { +[[maybe_unused]] thread_local bool avoid_logging; +} namespace { enum class split_str_mode { is_any_of, @@ -490,6 +493,10 @@ logger::log_syscall(syscall::info info, const long syscall_number, print_syscall: + if(::avoid_logging) { + return; + } + ::avoid_logging = true; fmt::basic_memory_buffer buffer; detail::format_timestamp_to(buffer, timezone_); @@ -504,6 +511,7 @@ print_syscall: fmt::format_to(std::back_inserter(buffer), "\n"); ::syscall_no_intercept(SYS_write, log_fd_, buffer.data(), buffer.size()); + ::avoid_logging = false; } } // namespace gkfs::log -- GitLab From 9c94ceee197122e46cb41ecab1e299542a9d9fa5 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Wed, 13 Mar 2024 15:42:02 +0100 Subject: [PATCH 10/27] Removing date dependency of the client --- include/client/logging.hpp | 148 +++++++++++++++++++++++++++++++++---- src/client/CMakeLists.txt | 12 ++- src/client/logging.cpp | 42 +---------- 3 files changed, 147 insertions(+), 55 deletions(-) diff --git a/include/client/logging.hpp b/include/client/logging.hpp index 3224f9ce2..9a75278e9 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -175,6 +175,138 @@ log_buffer(int fd, const void* buffer, std::size_t length) { ::syscall_no_intercept(SYS_write, fd, buffer, length); } +/** + * @brief convert a time_t to a tm + * It is not POSIX compliant, but it dows not uses any syscall or timezone + * Converts a Unix timestamp (number of seconds since the beginning of 1970 + * CE) to a Gregorian civil date-time tuple in GMT (UTC) time zone. + * + * This conforms to C89 (and C99...) and POSIX. + * + * This implementation works, and doesn't overflow for any sizeof(time_t). + * It doesn't check for overflow/underflow in tm->tm_year output. Other than + * that, it never overflows or underflows. It assumes that that time_t is + * signed. + * + * This implements the inverse of the POSIX formula + * (http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_15) + * for all time_t values, no matter the size, as long as tm->tm_year doesn't + * overflow or underflow. The formula is: tm_sec + tm_min*60 + tm_hour*3600 + * + tm_yday*86400 + (tm_year-70)*31536000 + ((tm_year-69)/4)*86400 - + * ((tm_year-1)/100)*86400 + ((tm_year+299)/400)*86400. + * + * License : GNU General Public License v2.0 from + * https://github.com/pts/minilibc686/ + * @param time_t + * @return tm + */ + +static inline struct tm* +mini_gmtime_r(const time_t* timep, struct tm* tm) { + const time_t ts = *timep; + time_t t = ts / 86400; + unsigned hms = + ts % + 86400; /* -86399 <= hms <= 86399. This needs sizeof(int) >= 4. */ + time_t c, f; + unsigned yday; /* 0 <= yday <= 426. Also fits to an `unsigned short', but + `int' is faster. */ + unsigned a; /* 0 <= a <= 2133. Also fits to an `unsigned short', but `int' + is faster. */ + if((int) hms < 0) { + --t; + hms += 86400; + } /* Fix quotient and negative remainder if ts was negative (i.e. before + year 1970 CE). */ + /* Now: -24856 <= t <= 24855. */ + tm->tm_sec = hms % 60; + hms /= 60; + tm->tm_min = hms % 60; + tm->tm_hour = hms / 60; + if(sizeof(time_t) > + 4) { /* Optimization. For int32_t, this would keep t intact, so we won't + have to do it. This produces unreachable code. */ + f = (t + 4) % 7; + if(f < 0) + f += 7; /* Fix negative remainder if (t + 4) was negative. */ + /* Now 0 <= f <= 6. */ + tm->tm_wday = f; + c = (t << 2) + 102032; + f = c / 146097; + if(c % 146097 < 0) + --f; /* Fix negative remainder if c was negative. */ + --f; + t += f; + f >>= 2; + t -= f; + f = (t << 2) + 102035; + c = f / 1461; + if(f % 1461 < 0) + --c; /* Fix negative remainder if f was negative. */ + } else { + tm->tm_wday = (t + 24861) % 7; /* t + 24861 >= 0. */ + /* Now: -24856 <= t <= 24855. */ + c = ((t << 2) + 102035) / 1461; + } + yday = t - 365 * c - (c >> 2) + 25568; + /* Now: 0 <= yday <= 425. */ + a = yday * 5 + 8; + /* Now: 8 <= a <= 2133. */ + tm->tm_mon = a / 153; + a %= 153; /* No need to fix if a < 0, because a cannot be negative here. */ + /* Now: 2 <= tm->tm_mon <= 13. */ + /* Now: 0 <= a <= 152. */ + tm->tm_mday = 1 + a / 5; /* No need to fix if a < 0, because a cannot be + negative here. */ + /* Now: 1 <= tm->tm_mday <= 31. */ + if(tm->tm_mon >= 12) { + tm->tm_mon -= 12; + /* Now: 0 <= tm->tm_mon <= 1. */ + ++c; + yday -= 366; + } else { /* Check for leap year (in c). */ + /* Now: 2 <= tm->tm_mon <= 11. */ + /* 1903: not leap; 1904: leap, 1900: not leap; 2000: leap */ + /* With sizeof(time_t) == 4, we have 1901 <= year <= 2038; of these + * years only 2000 is divisble by 100, and that's a leap year, no we + * optimize the check to `(c & 3) == 0' only. + */ + if(!((c & 3) == 0 && + (sizeof(time_t) <= 4 || c % 100 != 0 || (c + 300) % 400 == 0))) + --yday; /* These `== 0' comparisons work even if c < 0. */ + } + tm->tm_year = + c; /* This assignment may overflow or underflow, we don't check it. + Example: time_t is a huge int64_t, tm->tm_year is int32_t. */ + /* Now: 0 <= tm->tm_mon <= 11. */ + /* Now: 0 <= yday <= 365. */ + tm->tm_yday = yday; + tm->tm_isdst = 0; + return tm; +} + +static inline struct tm* +mini_gmtime(const time_t* timep) { + static struct tm tm; + return mini_gmtime_r(timep, &tm); +} + +static inline ssize_t +format_timeval(struct timeval* tv, char* buf, size_t sz) { + ssize_t written = -1; + struct tm* gm = mini_gmtime(&tv->tv_sec); + + + written = (ssize_t) strftime(buf, sz, "%Y-%m-%d %H:%M:%S", gm); + if((written > 0) && ((size_t) written < sz)) { + int w = snprintf(buf + written, sz - (size_t) written, ".%06ld", + tv->tv_usec); + written = (w > 0) ? written + w : -1; + } + + return written; +} + /** * format_timestamp_to - safely format a timestamp for logging messages * @@ -208,21 +340,11 @@ format_timestamp_to(Buffer&& buffer, return; } - date::sys_time now{ - std::chrono::seconds{tv.tv_sec} + - std::chrono::microseconds{tv.tv_usec}}; + char buf[28]; - if(!timezone) { - fmt::format_to(std::back_inserter(buffer), "[{}] ", - now.time_since_epoch().count()); - return; + if(format_timeval(&tv, buf, sizeof(buf)) > 0) { + fmt::format_to(std::back_inserter(buffer), "[{}] ", buf); } - - auto str = - format("%F %T", - date::zoned_time{timezone, now}); - - fmt::format_to(std::back_inserter(buffer), "[{}] ", str); } template diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 3055c3e92..333fd684e 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -66,7 +66,17 @@ target_link_libraries( hermes fmt::fmt Threads::Threads - Date::TZ + Syscall_intercept::Syscall_intercept +) + +target_link_libraries( + gkfs_user_lib + PRIVATE metadata distributor env_util arithmetic path_util rpc_utils + PUBLIC dl + Mercury::Mercury + hermes + fmt::fmt + Threads::Threads ) install( diff --git a/src/client/logging.cpp b/src/client/logging.cpp index e8a243e45..612bed825 100644 --- a/src/client/logging.cpp +++ b/src/client/logging.cpp @@ -32,11 +32,7 @@ #include #include #include - -extern "C" { -#include -#include -} +#include #ifdef GKFS_ENABLE_LOGGING @@ -46,9 +42,6 @@ extern "C" { namespace fs = std::filesystem; -namespace { -[[maybe_unused]] thread_local bool avoid_logging; -} namespace { enum class split_str_mode { is_any_of, @@ -338,34 +331,6 @@ logger::logger(const std::string& opts, const std::string& path, log_fd_ = fd; } - // Finding the current timezone implies accessing OS files (i.e. syscalls), - // but current_zone() doesn't actually retrieve the time zone but rather - // provides a descriptor to it that is **atomically initialized** upon its - // first use. Thus, if we don't force the initialization here, logging the - // first intercepted syscall will produce a call to date::time_zone::init() - // (under std::call_once) which internally ends up calling fopen(). Since - // fopen() ends up calling sys_open(), we will need to generate another - // timestamp for a system call log entry, which will attempt to call - // date::time_zone::init() since the prior initialization (under the same - // std::call_once) has not yet completed. - // - // Unfortunately, date::time_zone doesn't provide a function to prevent - // this lazy initialization, therefore we force it by requesting - // information from an arbitrary timepoint (January 1st 1970) which forces - // the initialization. This doesn't do any actual work and could safely be - // removed if the date API ends up providing this functionality. - try { - timezone_ = date::current_zone(); -#ifdef GKFS_DEBUG_BUILD - using namespace date; - timezone_->get_info(date::sys_days{January / 1 / 1970}); -#endif // GKFS_DEBUG_BUILD - } catch(const std::exception& ex) { - // if timezone initialization fails, setting timezone_ to nullptr - // makes format_timestamp_to() default to producing epoch timestamps - timezone_ = nullptr; - } - #ifdef GKFS_ENABLE_LOGGING const auto log_hermes_message = [](const std::string& msg, hermes::log::level l, int severity, @@ -493,10 +458,6 @@ logger::log_syscall(syscall::info info, const long syscall_number, print_syscall: - if(::avoid_logging) { - return; - } - ::avoid_logging = true; fmt::basic_memory_buffer buffer; detail::format_timestamp_to(buffer, timezone_); @@ -511,7 +472,6 @@ print_syscall: fmt::format_to(std::back_inserter(buffer), "\n"); ::syscall_no_intercept(SYS_write, log_fd_, buffer.data(), buffer.size()); - ::avoid_logging = false; } } // namespace gkfs::log -- GitLab From 8b3267aed05bf2589390a08a1c040853fd1a3217 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 20 Oct 2023 12:20:56 +0200 Subject: [PATCH 11/27] Test updates, Changelog --- .gitmodules | 2 +- CHANGELOG.md | 1 + CMake/FindDate.cmake | 68 ------------------- CMakeLists.txt | 4 -- scripts/profiles/0.9.3/agios.specs | 3 +- scripts/profiles/0.9.3/all.specs | 3 +- scripts/profiles/0.9.3/arm.specs | 3 +- scripts/profiles/0.9.3/ci.specs | 3 +- scripts/profiles/0.9.3/default.specs | 3 +- scripts/profiles/0.9.3/infiniband_verbs.specs | 3 +- scripts/profiles/0.9.3/install/date.install | 63 ----------------- scripts/profiles/0.9.3/marenostrum4.specs | 3 +- scripts/profiles/0.9.3/mogon2.specs | 3 +- scripts/profiles/0.9.3/ngio.specs | 3 +- scripts/profiles/0.9.3/omnipath_psm2.specs | 3 +- scripts/profiles/0.9.3/p9.specs | 3 +- tests/scripts/compile_dep.sh/0.9.3/agios.out | 1 - tests/scripts/compile_dep.sh/0.9.3/all.out | 1 - tests/scripts/compile_dep.sh/0.9.3/arm.out | 1 - tests/scripts/compile_dep.sh/0.9.3/ci.out | 1 - .../scripts/compile_dep.sh/0.9.3/default.out | 2 +- .../compile_dep.sh/0.9.3/infiniband_verbs.out | 1 - .../compile_dep.sh/0.9.3/marenostrum4.out | 1 - tests/scripts/compile_dep.sh/0.9.3/mogon2.out | 1 - tests/scripts/compile_dep.sh/0.9.3/ngio.out | 1 - .../compile_dep.sh/0.9.3/omnipath_psm2.out | 1 - tests/scripts/compile_dep.sh/0.9.3/p9.out | 1 - tests/scripts/dl_dep.sh/0.9.3/agios.out | 1 - tests/scripts/dl_dep.sh/0.9.3/all.out | 3 +- tests/scripts/dl_dep.sh/0.9.3/arm.out | 3 +- tests/scripts/dl_dep.sh/0.9.3/ci.out | 3 +- tests/scripts/dl_dep.sh/0.9.3/default.out | 3 +- tests/scripts/dl_dep.sh/0.9.3/direct.out | 2 +- .../dl_dep.sh/0.9.3/infiniband_verbs.out | 3 +- .../scripts/dl_dep.sh/0.9.3/marenostrum4.out | 3 +- tests/scripts/dl_dep.sh/0.9.3/mogon2.out | 3 +- tests/scripts/dl_dep.sh/0.9.3/ngio.out | 3 +- .../scripts/dl_dep.sh/0.9.3/omnipath_psm2.out | 3 +- tests/scripts/dl_dep.sh/0.9.3/p9.out | 3 +- 39 files changed, 25 insertions(+), 191 deletions(-) delete mode 100644 CMake/FindDate.cmake delete mode 100644 scripts/profiles/0.9.3/install/date.install diff --git a/.gitmodules b/.gitmodules index 2d710ba8d..dec4e3a46 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "external/hermes"] path = external/hermes url = https://github.com/bsc-ssrg/hermes.git - commit = 9f17ff97fd0b6aa44227cc164e099f25f3e5c333 + commit = bd4361897eba6e8acaab2d643f77f1a40c250f1d [submodule "tests/scripts/bats"] path = tests/scripts/bats url = https://github.com/bats-core/bats-core.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 589cf59ae..93e0446d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141) - Added reattemp support in get_fs_config to other servers, when the initial server fails. - Fused GekkoFWD and GekkoFS. GekkoFWD is enabled with the `--enable-following` in the server configuration and the ENV variable `LIBGKFS_FORWARDING_MAP_FILE` in the clients. ([!170](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/170)). +- FMT10 and date removal, several dependencies updated. ([!172](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/172)). ### New ### Changed diff --git a/CMake/FindDate.cmake b/CMake/FindDate.cmake deleted file mode 100644 index 44d6d12ad..000000000 --- a/CMake/FindDate.cmake +++ /dev/null @@ -1,68 +0,0 @@ -################################################################################ -# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # -# # -# This software was partially supported by the # -# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # -# # -# This software was partially supported by the # -# ADA-FS project under the SPPEXA project funded by the DFG. # -# # -# This file is part of GekkoFS. # -# # -# GekkoFS is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# GekkoFS is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with GekkoFS. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ - -find_path(DATE_INCLUDE_DIR - NAMES date/date.h -) - -find_path(TZ_INCLUDE_DIR - NAMES date/tz.h -) - -find_library(TZ_LIBRARY - NAMES date-tz -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( Date - DEFAULT_MSG - DATE_INCLUDE_DIR - TZ_INCLUDE_DIR - TZ_LIBRARY -) - -if(Date_FOUND) - set(DATE_INCLUDE_DIRS ${DATE_INCLUDE_DIR}) - set(TZ_INCLUDE_DIRS ${TZ_INCLUDE_DIR}) - set(TZ_LIBRARIES ${TZ_LIBRARY}) - - if(NOT TARGET Date::TZ) - add_library(Date::TZ UNKNOWN IMPORTED) - set_target_properties(Date::TZ PROPERTIES - IMPORTED_LOCATION "${TZ_LIBRARY}" - INTERFACE_COMPILE_DEFINITIONS "USE_OS_TZDB=1" - INTERFACE_INCLUDE_DIRECTORIES "${TZ_INCLUDE_DIR}" - ) - endif() -endif() - -mark_as_advanced( - DATE_INCLUDE_DIR - TZ_INCLUDE_DIR - TZ_LIBRARY -) diff --git a/CMakeLists.txt b/CMakeLists.txt index cc5450476..c70d36115 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -160,10 +160,6 @@ find_package(Margo 0.9.6 REQUIRED) message(STATUS "[${PROJECT_NAME}] Checking for syscall_intercept") find_package(Syscall_intercept REQUIRED) -### Date (required by the libgkfs_intercept.so) -message(STATUS "[${PROJECT_NAME}] Checking for Date") -find_package(Date REQUIRED) - ### AGIOS: required for scheduling I/O requests if (GKFS_ENABLE_AGIOS) message(STATUS "[${PROJECT_NAME}] Checking for Agios") diff --git a/scripts/profiles/0.9.3/agios.specs b/scripts/profiles/0.9.3/agios.specs index 0ebf47e8c..406adb383 100644 --- a/scripts/profiles/0.9.3/agios.specs +++ b/scripts/profiles/0.9.3/agios.specs @@ -50,7 +50,6 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" ) @@ -66,7 +65,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "agios" + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "agios" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.3/all.specs b/scripts/profiles/0.9.3/all.specs index 0909c9aff..9780acaa0 100644 --- a/scripts/profiles/0.9.3/all.specs +++ b/scripts/profiles/0.9.3/all.specs @@ -53,7 +53,6 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" ) @@ -70,7 +69,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "agios" "curl" "prometheus-cpp" "parallax" ) diff --git a/scripts/profiles/0.9.3/arm.specs b/scripts/profiles/0.9.3/arm.specs index 395c43610..a80ae6214 100644 --- a/scripts/profiles/0.9.3/arm.specs +++ b/scripts/profiles/0.9.3/arm.specs @@ -51,7 +51,6 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="fb73c3924b502e2840a5dc8a18746e395b06a8a6" - ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" ) # Extra arguments for git clone @@ -66,7 +65,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.3/ci.specs b/scripts/profiles/0.9.3/ci.specs index 2f2f22f10..44c847abc 100644 --- a/scripts/profiles/0.9.3/ci.specs +++ b/scripts/profiles/0.9.3/ci.specs @@ -48,7 +48,6 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" ) @@ -66,7 +65,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" - "date" "agios" "parallax" "prometheus-cpp" + "agios" "parallax" "prometheus-cpp" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.3/default.specs b/scripts/profiles/0.9.3/default.specs index 821c93b7b..d214eb0ae 100644 --- a/scripts/profiles/0.9.3/default.specs +++ b/scripts/profiles/0.9.3/default.specs @@ -50,7 +50,6 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" ) # Extra arguments for git clone @@ -65,7 +64,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.3/infiniband_verbs.specs b/scripts/profiles/0.9.3/infiniband_verbs.specs index a57dd217e..488957246 100644 --- a/scripts/profiles/0.9.3/infiniband_verbs.specs +++ b/scripts/profiles/0.9.3/infiniband_verbs.specs @@ -50,7 +50,6 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" ) # Extra arguments for git clone @@ -66,7 +65,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading/installing order=( "lz4" "capstone" "json-c" "libfabric%verbs" "mercury" "argobots" "margo" "rocksdb" - "syscall_intercept" "date" + "syscall_intercept" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.3/install/date.install b/scripts/profiles/0.9.3/install/date.install deleted file mode 100644 index 7e5dd6b31..000000000 --- a/scripts/profiles/0.9.3/install/date.install +++ /dev/null @@ -1,63 +0,0 @@ -################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # -# # -# This software was partially supported by the # -# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # -# # -# This software was partially supported by the # -# ADA-FS project under the SPPEXA project funded by the DFG. # -# # -# This file is part of GekkoFS. # -# # -# GekkoFS is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# GekkoFS is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with GekkoFS. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ -# vi: ft=bash - -################################################################################ -## The installation script must define both a pkg_install function and -## pkg_check function that, as their name implies, must specify how -## a dependency package should be installed and tested. ## ## The following -## variables can be used in the installation script: -## - CMAKE: a variable that expands to the cmake binary -## - SOURCE_DIR: the directory where the sources for the package were -## downloaded -## - INSTALL_DIR: the directory where the package should be installed -## - CORES: the number of cores to use when building -## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) -## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) -## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) -## - PERFORM_TEST: whether tests for the package should be executed -################################################################################ - -pkg_install() { - ID="date" - CURR="${SOURCE_DIR}/${ID}" - prepare_build_dir "${CURR}" - cd "${CURR}/build" - ${CMAKE} -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ - -DCMAKE_BUILD_TYPE:STRING=Release \ - -DCMAKE_CXX_STANDARD:STRING=14 \ - -DBUILD_TZ_LIB:BOOL=ON \ - -DUSE_SYSTEM_TZ_DB:BOOL=ON \ - -DBUILD_SHARED_LIBS:BOOL=ON .. - - make -j"${CORES}" install -} - -pkg_check() { - : -} diff --git a/scripts/profiles/0.9.3/marenostrum4.specs b/scripts/profiles/0.9.3/marenostrum4.specs index 20598044d..7df096a05 100644 --- a/scripts/profiles/0.9.3/marenostrum4.specs +++ b/scripts/profiles/0.9.3/marenostrum4.specs @@ -51,7 +51,6 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) @@ -67,7 +66,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "parallax" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.3/mogon2.specs b/scripts/profiles/0.9.3/mogon2.specs index 3b33fbad4..0f34f3ff6 100644 --- a/scripts/profiles/0.9.3/mogon2.specs +++ b/scripts/profiles/0.9.3/mogon2.specs @@ -51,7 +51,6 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) @@ -67,7 +66,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "parallax" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.3/ngio.specs b/scripts/profiles/0.9.3/ngio.specs index 7a73e8e40..3eab0bba2 100644 --- a/scripts/profiles/0.9.3/ngio.specs +++ b/scripts/profiles/0.9.3/ngio.specs @@ -51,7 +51,6 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) @@ -67,7 +66,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "parallax" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.3/omnipath_psm2.specs b/scripts/profiles/0.9.3/omnipath_psm2.specs index 66f62bb10..624185f9f 100644 --- a/scripts/profiles/0.9.3/omnipath_psm2.specs +++ b/scripts/profiles/0.9.3/omnipath_psm2.specs @@ -51,7 +51,6 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" ) # Extra arguments for git clone @@ -66,7 +65,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.3/p9.specs b/scripts/profiles/0.9.3/p9.specs index efa069ebc..6b646f5a3 100644 --- a/scripts/profiles/0.9.3/p9.specs +++ b/scripts/profiles/0.9.3/p9.specs @@ -51,7 +51,6 @@ clonedeps=( ["mercury"]="v2.1.0" ["margo"]="v0.9.6" ["syscall_intercept"]="6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f" - ["date"]="6e921e1b1d21e84a5c82416ba7ecd98e33a436d0" ) # Extra arguments for git clone @@ -66,7 +65,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" ) # Extra arguments passed to the installation script. As such, they can diff --git a/tests/scripts/compile_dep.sh/0.9.3/agios.out b/tests/scripts/compile_dep.sh/0.9.3/agios.out index b098aecfa..25a1cad3a 100644 --- a/tests/scripts/compile_dep.sh/0.9.3/agios.out +++ b/tests/scripts/compile_dep.sh/0.9.3/agios.out @@ -7,6 +7,5 @@ ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### ######## Installing: agios ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/all.out b/tests/scripts/compile_dep.sh/0.9.3/all.out index 5cdf4c5ff..2fcd55b6f 100644 --- a/tests/scripts/compile_dep.sh/0.9.3/all.out +++ b/tests/scripts/compile_dep.sh/0.9.3/all.out @@ -9,7 +9,6 @@ WARNING: Install script for 'psm2' not found. Skipping. ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### ######## Installing: agios ############################### ######## Installing: parallax ############################### Done \ No newline at end of file diff --git a/tests/scripts/compile_dep.sh/0.9.3/arm.out b/tests/scripts/compile_dep.sh/0.9.3/arm.out index 7138da9e2..e05da6537 100644 --- a/tests/scripts/compile_dep.sh/0.9.3/arm.out +++ b/tests/scripts/compile_dep.sh/0.9.3/arm.out @@ -9,4 +9,3 @@ WARNING: Install script for 'psm2' not found. Skipping. ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/ci.out b/tests/scripts/compile_dep.sh/0.9.3/ci.out index d3c276800..9ec3c24bb 100644 --- a/tests/scripts/compile_dep.sh/0.9.3/ci.out +++ b/tests/scripts/compile_dep.sh/0.9.3/ci.out @@ -4,6 +4,5 @@ ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### ######## Installing: agios ############################### ######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/default.out b/tests/scripts/compile_dep.sh/0.9.3/default.out index 787e4a08f..ca393e65d 100644 --- a/tests/scripts/compile_dep.sh/0.9.3/default.out +++ b/tests/scripts/compile_dep.sh/0.9.3/default.out @@ -7,4 +7,4 @@ ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### + diff --git a/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out b/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out index e82dd020f..ff149d28f 100644 --- a/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out +++ b/tests/scripts/compile_dep.sh/0.9.3/infiniband_verbs.out @@ -7,4 +7,3 @@ ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out b/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out index 38f67ce3d..1af91c3cf 100644 --- a/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out +++ b/tests/scripts/compile_dep.sh/0.9.3/marenostrum4.out @@ -9,5 +9,4 @@ WARNING: Install script for 'psm2' not found. Skipping. ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### ######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/mogon2.out b/tests/scripts/compile_dep.sh/0.9.3/mogon2.out index 38f67ce3d..1af91c3cf 100644 --- a/tests/scripts/compile_dep.sh/0.9.3/mogon2.out +++ b/tests/scripts/compile_dep.sh/0.9.3/mogon2.out @@ -9,5 +9,4 @@ WARNING: Install script for 'psm2' not found. Skipping. ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### ######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/ngio.out b/tests/scripts/compile_dep.sh/0.9.3/ngio.out index 38f67ce3d..1af91c3cf 100644 --- a/tests/scripts/compile_dep.sh/0.9.3/ngio.out +++ b/tests/scripts/compile_dep.sh/0.9.3/ngio.out @@ -9,5 +9,4 @@ WARNING: Install script for 'psm2' not found. Skipping. ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### ######## Installing: parallax ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out b/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out index 7138da9e2..e05da6537 100644 --- a/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out +++ b/tests/scripts/compile_dep.sh/0.9.3/omnipath_psm2.out @@ -9,4 +9,3 @@ WARNING: Install script for 'psm2' not found. Skipping. ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### diff --git a/tests/scripts/compile_dep.sh/0.9.3/p9.out b/tests/scripts/compile_dep.sh/0.9.3/p9.out index 7138da9e2..e05da6537 100644 --- a/tests/scripts/compile_dep.sh/0.9.3/p9.out +++ b/tests/scripts/compile_dep.sh/0.9.3/p9.out @@ -9,4 +9,3 @@ WARNING: Install script for 'psm2' not found. Skipping. ######## Installing: margo ############################### ######## Installing: rocksdb ############################### ######## Installing: syscall_intercept ############################### -######## Installing: date ############################### diff --git a/tests/scripts/dl_dep.sh/0.9.3/agios.out b/tests/scripts/dl_dep.sh/0.9.3/agios.out index 1606211fa..8d8e2486b 100644 --- a/tests/scripts/dl_dep.sh/0.9.3/agios.out +++ b/tests/scripts/dl_dep.sh/0.9.3/agios.out @@ -7,6 +7,5 @@ Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobot Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/all.out b/tests/scripts/dl_dep.sh/0.9.3/all.out index 4a1ab9e6f..dbd218318 100644 --- a/tests/scripts/dl_dep.sh/0.9.3/all.out +++ b/tests/scripts/dl_dep.sh/0.9.3/all.out @@ -6,9 +6,8 @@ Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEA Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/arm.out b/tests/scripts/dl_dep.sh/0.9.3/arm.out index 2c61edb9c..7b1d7c5fc 100644 --- a/tests/scripts/dl_dep.sh/0.9.3/arm.out +++ b/tests/scripts/dl_dep.sh/0.9.3/arm.out @@ -6,7 +6,6 @@ Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEA Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[fb73c3924b502e2840a5dc8a18746e395b06a8a6]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Done diff --git a/tests/scripts/dl_dep.sh/0.9.3/ci.out b/tests/scripts/dl_dep.sh/0.9.3/ci.out index 4ee089ef7..0754a2860 100644 --- a/tests/scripts/dl_dep.sh/0.9.3/ci.out +++ b/tests/scripts/dl_dep.sh/0.9.3/ci.out @@ -2,9 +2,8 @@ Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEA Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/default.out b/tests/scripts/dl_dep.sh/0.9.3/default.out index a68701cf0..7b7330872 100644 --- a/tests/scripts/dl_dep.sh/0.9.3/default.out +++ b/tests/scripts/dl_dep.sh/0.9.3/default.out @@ -5,7 +5,6 @@ Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEA Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/direct.out b/tests/scripts/dl_dep.sh/0.9.3/direct.out index e27eae530..42a6d1545 100644 --- a/tests/scripts/dl_dep.sh/0.9.3/direct.out +++ b/tests/scripts/dl_dep.sh/0.9.3/direct.out @@ -2,6 +2,6 @@ Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEA Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out b/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out index 6ebc289ff..f2c3f6d3b 100644 --- a/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out +++ b/tests/scripts/dl_dep.sh/0.9.3/infiniband_verbs.out @@ -5,7 +5,6 @@ Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out index 657154fdc..f2bc30bb7 100644 --- a/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out +++ b/tests/scripts/dl_dep.sh/0.9.3/marenostrum4.out @@ -6,8 +6,7 @@ Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEA Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/mogon2.out b/tests/scripts/dl_dep.sh/0.9.3/mogon2.out index 657154fdc..f2bc30bb7 100644 --- a/tests/scripts/dl_dep.sh/0.9.3/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.3/mogon2.out @@ -6,8 +6,7 @@ Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEA Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/ngio.out b/tests/scripts/dl_dep.sh/0.9.3/ngio.out index 657154fdc..f2bc30bb7 100644 --- a/tests/scripts/dl_dep.sh/0.9.3/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.3/ngio.out @@ -6,8 +6,7 @@ Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEA Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/CARV-ICS-FORTH/parallax.git' to 'parallax' with commit '[c130decd7a71c60c20b98d6a23924f05f754c3cd]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out index d26b498ca..8ef7035e0 100644 --- a/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.3/omnipath_psm2.out @@ -6,7 +6,6 @@ Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEA Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[2c8765fa292bc9c28a22624c528580d54658813d]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Done \ No newline at end of file diff --git a/tests/scripts/dl_dep.sh/0.9.3/p9.out b/tests/scripts/dl_dep.sh/0.9.3/p9.out index 3a99bb523..1c939fc23 100644 --- a/tests/scripts/dl_dep.sh/0.9.3/p9.out +++ b/tests/scripts/dl_dep.sh/0.9.3/p9.out @@ -6,7 +6,6 @@ Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEA Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.9.6]' and flags '' -Downloaded 'https://github.com/facebook/rocksdb/archive/v6.26.1.tar.gz' to 'rocksdb' +Downloaded 'https://github.com/facebook/rocksdb/archive/v8.6.7.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f]' and flags '' -Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Done -- GitLab From ad767d172f27f2f60394fbaa78e2c0def50956cc Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 24 Nov 2023 18:17:42 +0100 Subject: [PATCH 12/27] Solved idx bug on reattemp forward_get_fs --- src/client/rpc/forward_management.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/rpc/forward_management.cpp b/src/client/rpc/forward_management.cpp index c975842e5..7504b1f6b 100644 --- a/src/client/rpc/forward_management.cpp +++ b/src/client/rpc/forward_management.cpp @@ -47,7 +47,7 @@ forward_get_fs_config() { bool found = false; size_t idx = 0; - while(!found && idx <= CTX->hosts().size()) { + while(!found && idx < CTX->hosts().size()) { try { LOG(DEBUG, "Retrieving file system configurations from daemon"); // TODO(amiranda): add a post() with RPC_TIMEOUT to hermes so that -- GitLab From 271b4a1041394262ecee2040e8484d84b08df02f Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 10 Jan 2024 11:18:49 +0100 Subject: [PATCH 13/27] removed old include date/tz --- include/client/logging.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/include/client/logging.hpp b/include/client/logging.hpp index 9a75278e9..fd6138c2f 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -38,7 +38,6 @@ #include #include #include -#include #include #ifdef GKFS_DEBUG_BUILD -- GitLab From 8699a0ab5ecc58fe39efbb071d75c377bb94dafa Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 10 Jan 2024 11:54:43 +0100 Subject: [PATCH 14/27] Removal of date leftovers --- include/client/logging.hpp | 9 ++------- src/client/logging.cpp | 5 ++--- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/include/client/logging.hpp b/include/client/logging.hpp index fd6138c2f..1293d70a4 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -323,13 +323,10 @@ format_timeval(struct timeval* tv, char* buf, size_t sz) { * one thread exactly, and we pass it as an argument whenever we need to * format a timestamp. If no timezone is provided, we just format the epoch. * - * NOTE: we use the date C++ library to query the timezone database and - * to format the timestamps. */ template static inline void -format_timestamp_to(Buffer&& buffer, - const date::time_zone* const timezone = nullptr) { +format_timestamp_to(Buffer&& buffer) { struct ::timeval tv; @@ -413,7 +410,7 @@ struct logger { } static_buffer buffer; - detail::format_timestamp_to(buffer, timezone_); + detail::format_timestamp_to(buffer); fmt::format_to(std::back_inserter(buffer), "[{}] [{}] ", , log_process_id_, lookup_level_name(level)); @@ -531,8 +528,6 @@ struct logger { std::bitset<512> filtered_syscalls_; int debug_verbosity_; #endif - - const date::time_zone* timezone_; }; // the following static functions can be used to interact diff --git a/src/client/logging.cpp b/src/client/logging.cpp index 612bed825..830d5691e 100644 --- a/src/client/logging.cpp +++ b/src/client/logging.cpp @@ -285,8 +285,7 @@ logger::logger(const std::string& opts, const std::string& path, , const std::string& filter, int verbosity #endif - ) - : timezone_(nullptr) { +) { /* use stderr by default */ log_fd_ = 2; @@ -460,7 +459,7 @@ print_syscall: fmt::basic_memory_buffer buffer; - detail::format_timestamp_to(buffer, timezone_); + detail::format_timestamp_to(buffer); detail::format_syscall_info_to(buffer, info); if(result) { -- GitLab From edcbcadef2e09122f43024f94bcf07817a85a81a Mon Sep 17 00:00:00 2001 From: rnou Date: Mon, 19 Feb 2024 11:45:18 +0100 Subject: [PATCH 15/27] extra comma on merge --- include/client/logging.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/client/logging.hpp b/include/client/logging.hpp index 1293d70a4..2f00058d0 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -411,7 +411,7 @@ struct logger { static_buffer buffer; detail::format_timestamp_to(buffer); - fmt::format_to(std::back_inserter(buffer), "[{}] [{}] ", , + fmt::format_to(std::back_inserter(buffer), "[{}] [{}] ", log_process_id_, lookup_level_name(level)); if(!!(level & log::debug)) { -- GitLab From 10d1b9d582b505af4199036c866bd3143c30c5db Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 14 Mar 2024 16:29:53 +0100 Subject: [PATCH 16/27] Fix submodules and CMake --- .gitmodules | 3 +-- src/client/CMakeLists.txt | 10 ---------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/.gitmodules b/.gitmodules index dec4e3a46..31675fc19 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,6 @@ [submodule "external/hermes"] path = external/hermes - url = https://github.com/bsc-ssrg/hermes.git - commit = bd4361897eba6e8acaab2d643f77f1a40c250f1d + url = https://github.com/gekkofs/hermes.git [submodule "tests/scripts/bats"] path = tests/scripts/bats url = https://github.com/bats-core/bats-core.git diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 333fd684e..c408fc726 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -69,16 +69,6 @@ target_link_libraries( Syscall_intercept::Syscall_intercept ) -target_link_libraries( - gkfs_user_lib - PRIVATE metadata distributor env_util arithmetic path_util rpc_utils - PUBLIC dl - Mercury::Mercury - hermes - fmt::fmt - Threads::Threads -) - install( TARGETS gkfs_intercept LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} -- GitLab From 7afe20d62cc0e83fc31e6f012259b8ed754156c7 Mon Sep 17 00:00:00 2001 From: rnou Date: Tue, 10 Oct 2023 12:38:22 +0200 Subject: [PATCH 17/27] Creating Gekko API example --- examples/gfind/CMakeLists.txt | 8 +++ examples/gfind/gkfs_lib_example.cpp | 102 ++++++++++++++++++++++++++++ include/client/gkfs_functions.hpp | 3 + include/client/preload.hpp | 5 ++ src/client/gkfs_functions.cpp | 21 ++++++ src/client/hooks.cpp | 13 +--- src/client/preload.cpp | 42 ++++++++++-- 7 files changed, 179 insertions(+), 15 deletions(-) create mode 100644 examples/gfind/gkfs_lib_example.cpp diff --git a/examples/gfind/CMakeLists.txt b/examples/gfind/CMakeLists.txt index d9e63e317..d46441f08 100644 --- a/examples/gfind/CMakeLists.txt +++ b/examples/gfind/CMakeLists.txt @@ -28,9 +28,17 @@ set (CMAKE_CXX_STANDARD 14) add_executable(sfind sfind.cpp) +add_executable(gkfs_lib_example gkfs_lib_example.cpp) + +target_link_libraries(gkfs_lib_example + PRIVATE gkfs_intercept +) if(GKFS_INSTALL_TESTS) install(TARGETS sfind DESTINATION ${CMAKE_INSTALL_BINDIR} ) + install(TARGETS gkfs_lib_example + DESTINATION ${CMAKE_INSTALL_BINDIR} + ) endif() diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/gfind/gkfs_lib_example.cpp new file mode 100644 index 000000000..b1ced51c3 --- /dev/null +++ b/examples/gfind/gkfs_lib_example.cpp @@ -0,0 +1,102 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +/* Function exported from GekkoFS LD_PRELOAD, code needs to be compiled with + * -fPIC */ +extern "C" int gkfs_init() + __attribute__((weak)); + +extern "C" int gkfs_end() + __attribute__((weak)); + +void init_preload() {}; +void destroy_preload() {}; + + + +void write_file(std::string filename){ + // Opem File + int fd = gkfs::syscall::gkfs_open(filename,S_IRWXU,O_RDWR|O_CREAT); + + cout << "FD open " << fd << endl; + char *buf = "testing"; + + // int size = gkfs::syscall::gkfs_write(fd, buf, 7); + + // cout << "FD size" << size << endl; + + gkfs::syscall::gkfs_close(fd); + +} + + +void read_file(std::string filename){ + int fdread = gkfs::syscall::gkfs_open(filename,S_IRWXU,O_RDONLY); + char *bufread = "TESTING\0"; + int sizeread = gkfs::syscall::gkfs_read(fdread, bufread, 7); + + cout << "Reading : " << sizeread << " --> " << bufread << endl; + + gkfs::syscall::gkfs_close(fdread); + +} +int main(int argc, char **argv) { + cout << "GekkoFS Client library test" << endl; + + auto res = gkfs_init(); + + cout << "Init result " << res << endl; + + //write_file ("/test.tmp"); + + read_file("/test.tmp"); + + res = gkfs_end(); + + cout << "End result " << res << endl; + +} diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 63b9b655f..574b084ee 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -149,6 +149,9 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, int gkfs_rmdir(const std::string& path); +int +gkfs_close(unsigned int fd); + #ifdef HAS_RENAME int gkfs_rename(const std::string& old_path, const std::string& new_path); diff --git a/include/client/preload.hpp b/include/client/preload.hpp index 6cae014b1..929e30eaf 100644 --- a/include/client/preload.hpp +++ b/include/client/preload.hpp @@ -47,5 +47,10 @@ init_preload() __attribute__((constructor)); void destroy_preload() __attribute__((destructor)); +extern "C" int +gkfs_init(); + +extern "C" int +gkfs_end(); #endif // IOINTERCEPT_PRELOAD_HPP diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index b03e8eaeb..ef63cd8d0 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1394,6 +1394,27 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, return written; } +/** + * @brief Closes an fd. To be used externally + * + * @param fd + * @return int + */ +int gkfs_close(unsigned int fd) { + if(CTX->file_map()->exist(fd)) { + // No call to the daemon is required + CTX->file_map()->remove(fd); + return 0; + } + + if(CTX->is_internal_fd(fd)) { + // the client application (for some reason) is trying to close an + // internal fd: ignore it + return 0; + } + + return -1; +} #ifdef HAS_SYMLINKS #ifdef GKFS_ENABLE_UNUSED_FUNCTIONS diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index a986d5898..ea7559c46 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -91,17 +91,10 @@ hook_close(int fd) { LOG(DEBUG, "{}() called with fd: {}", __func__, fd); - if(CTX->file_map()->exist(fd)) { - // No call to the daemon is required - CTX->file_map()->remove(fd); + auto ret = gkfs::syscall::gkfs_close(fd); + + if (ret == 0) return 0; - } - - if(CTX->is_internal_fd(fd)) { - // the client application (for some reason) is trying to close an - // internal fd: ignore it - return 0; - } return syscall_no_intercept_wrapper(SYS_close, fd); } diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 6d7a0c0e9..589656796 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -67,8 +67,10 @@ exit_error_msg(int errcode, const string& msg) { // if we don't disable interception before calling ::exit() // syscall hooks may find an inconsistent in shared state // (e.g. the logger) and thus, crash - gkfs::preload::stop_interception(); - CTX->disable_interception(); + if(CTX->interception_enabled()) { + gkfs::preload::stop_interception(); + CTX->disable_interception(); + } ::exit(errcode); } @@ -314,9 +316,39 @@ destroy_preload() { ld_network_service.reset(); LOG(DEBUG, "RPC subsystem shut down"); - gkfs::preload::stop_interception(); - CTX->disable_interception(); - LOG(DEBUG, "Syscall interception stopped"); + if(CTX->interception_enabled()) { + gkfs::preload::stop_interception(); + CTX->disable_interception(); + LOG(DEBUG, "Syscall interception stopped"); + } LOG(INFO, "All subsystems shut down. Client shutdown complete."); } + + + +/** + * @brief External functions to call linking the library + * + */ +extern "C" int +gkfs_init() { + CTX->init_logging(); + // from here ownwards it is safe to print messages + LOG(DEBUG, "Logging subsystem initialized"); + gkfs::preload::init_environment(); + return 4; +} + + +extern "C" int +gkfs_end() { + CTX->clear_hosts(); + LOG(DEBUG, "Peer information deleted"); + + ld_network_service.reset(); + LOG(DEBUG, "RPC subsystem shut down"); + + LOG(INFO, "All subsystems shut down. Client shutdown complete."); + return 4; +} \ No newline at end of file -- GitLab From 17a5a1e5aa05737dfda1d5966c43d9e6ec70a690 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 14 Mar 2024 16:34:04 +0100 Subject: [PATCH 18/27] 2 libs, disable syscall_intercept --- examples/gfind/CMakeLists.txt | 2 +- examples/gfind/gkfs_lib_example.cpp | 2 -- src/client/CMakeLists.txt | 38 +++++++++++++++++++++ src/client/void_syscall_intercept.cpp | 49 +++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 src/client/void_syscall_intercept.cpp diff --git a/examples/gfind/CMakeLists.txt b/examples/gfind/CMakeLists.txt index d46441f08..22c9bfa99 100644 --- a/examples/gfind/CMakeLists.txt +++ b/examples/gfind/CMakeLists.txt @@ -31,7 +31,7 @@ add_executable(sfind sfind.cpp) add_executable(gkfs_lib_example gkfs_lib_example.cpp) target_link_libraries(gkfs_lib_example - PRIVATE gkfs_intercept + PRIVATE gkfs_user_lib ) if(GKFS_INSTALL_TESTS) diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/gfind/gkfs_lib_example.cpp index b1ced51c3..3ed6e301b 100644 --- a/examples/gfind/gkfs_lib_example.cpp +++ b/examples/gfind/gkfs_lib_example.cpp @@ -56,8 +56,6 @@ extern "C" int gkfs_end() void init_preload() {}; void destroy_preload() {}; - - void write_file(std::string filename){ // Opem File int fd = gkfs::syscall::gkfs_open(filename,S_IRWXU,O_RDWR|O_CREAT); diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index c408fc726..7961f8464 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -32,6 +32,7 @@ # based on syscall interception. # ############################################################################## add_library(gkfs_intercept SHARED) +add_library (gkfs_user_lib SHARED) target_sources( gkfs_intercept @@ -52,6 +53,25 @@ target_sources( syscalls/detail/syscall_info.c ) +target_sources( + gkfs_user_lib + PRIVATE void_syscall_intercept.cpp + gkfs_functions.cpp + hooks.cpp + intercept.cpp + logging.cpp + open_file_map.cpp + open_dir.cpp + path.cpp + preload.cpp + preload_context.cpp + preload_util.cpp + rpc/rpc_types.cpp + rpc/forward_data.cpp + rpc/forward_management.cpp + rpc/forward_metadata.cpp + syscalls/detail/syscall_info.c +) if(GKFS_ENABLE_AGIOS) target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) @@ -69,6 +89,16 @@ target_link_libraries( Syscall_intercept::Syscall_intercept ) +target_link_libraries( + gkfs_user_lib + PRIVATE metadata distributor env_util arithmetic path_util rpc_utils + PUBLIC dl + Mercury::Mercury + hermes + fmt::fmt + Threads::Threads +) + install( TARGETS gkfs_intercept LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} @@ -76,3 +106,11 @@ install( PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) + +install( + TARGETS gkfs_user_lib + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs +) + diff --git a/src/client/void_syscall_intercept.cpp b/src/client/void_syscall_intercept.cpp new file mode 100644 index 000000000..744dd7195 --- /dev/null +++ b/src/client/void_syscall_intercept.cpp @@ -0,0 +1,49 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS' POSIX interface. + + GekkoFS' POSIX interface is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GekkoFS' POSIX interface is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with GekkoFS' POSIX interface. If not, see + . + + SPDX-License-Identifier: LGPL-3.0-or-later +*/ +extern "C" { +int (*intercept_hook_point)(long syscall_number, + long arg0, long arg1, + long arg2, long arg3, + long arg4, long arg5, + long *result) {}; +void (*intercept_hook_point_clone_child)( + unsigned long flags, void *child_stack, + int *ptid, int *ctid, long newtls) {}; +void (*intercept_hook_point_clone_parent)( + unsigned long flags, void *child_stack, + int *ptid, int *ctid, long newtls, + long returned_pid) {}; + void (*intercept_hook_point_post_kernel)(long syscall_number, + long arg0, long arg1, + long arg2, long arg3, + long arg4, long arg5, + long result) {}; + +long syscall_no_intercept(long syscall_number, ...) { return 0}; +} \ No newline at end of file -- GitLab From 7955a5f6d84b4cb27d54e6bf89fd5e1fb3837696 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 11 Oct 2023 09:45:53 +0200 Subject: [PATCH 19/27] Library finish --- CMakeLists.txt | 13 ++-- examples/gfind/gkfs_lib_example.cpp | 66 ++++++++++--------- include/client/CMakeLists.txt | 26 ++++++++ include/client/gkfs_functions.hpp | 1 + include/client/hooks.hpp | 6 ++ include/client/logging.hpp | 4 ++ .../client/void_syscall_intercept.hpp | 36 +++++----- src/client/CMakeLists.txt | 34 +++++----- src/client/gkfs_functions.cpp | 11 ++-- src/client/hooks.cpp | 4 +- src/client/intercept.cpp | 18 +++++ src/client/path.cpp | 7 +- src/client/preload.cpp | 9 ++- src/client/preload_context.cpp | 8 ++- src/client/syscalls/util.S | 49 ++++++++++++++ 15 files changed, 202 insertions(+), 90 deletions(-) rename src/client/void_syscall_intercept.cpp => include/client/void_syscall_intercept.hpp (64%) create mode 100644 src/client/syscalls/util.S diff --git a/CMakeLists.txt b/CMakeLists.txt index c70d36115..cda0d59e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,16 +32,17 @@ cmake_minimum_required(VERSION 3.13) project( GekkoFS VERSION 0.9.2 + LANGUAGES ASM CXX C ) enable_testing() -if (NOT CMAKE_COMPILER_IS_GNUCC) - message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") -endif () -if (NOT CMAKE_COMPILER_IS_GNUCXX) - message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported") -endif () +#if (NOT CMAKE_COMPILER_IS_GNUCC) + # message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") +#endif () +#if (NOT CMAKE_COMPILER_IS_GNUCXX) +# message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported") +#endif () set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/gfind/gkfs_lib_example.cpp index 3ed6e301b..513b0ef98 100644 --- a/examples/gfind/gkfs_lib_example.cpp +++ b/examples/gfind/gkfs_lib_example.cpp @@ -47,54 +47,58 @@ using namespace std; /* Function exported from GekkoFS LD_PRELOAD, code needs to be compiled with * -fPIC */ -extern "C" int gkfs_init() - __attribute__((weak)); +extern "C" int +gkfs_init() __attribute__((weak)); -extern "C" int gkfs_end() - __attribute__((weak)); +extern "C" int +gkfs_end() __attribute__((weak)); -void init_preload() {}; -void destroy_preload() {}; +void +init_preload(){}; +void +destroy_preload(){}; -void write_file(std::string filename){ - // Opem File - int fd = gkfs::syscall::gkfs_open(filename,S_IRWXU,O_RDWR|O_CREAT); +void +write_file(std::string filename) { + // Open File + int fd = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDWR | O_CREAT); - cout << "FD open " << fd << endl; - char *buf = "testing"; - - // int size = gkfs::syscall::gkfs_write(fd, buf, 7); + cout << "FD open " << fd << endl; + char* buf = "testing"; - // cout << "FD size" << size << endl; + int size = gkfs::syscall::gkfs_write(fd, buf, 7); - gkfs::syscall::gkfs_close(fd); + cout << "FD size" << size << endl; + gkfs::syscall::gkfs_close(fd); } -void read_file(std::string filename){ - int fdread = gkfs::syscall::gkfs_open(filename,S_IRWXU,O_RDONLY); - char *bufread = "TESTING\0"; - int sizeread = gkfs::syscall::gkfs_read(fdread, bufread, 7); +void +read_file(std::string filename) { + int fdread = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDONLY); + if(fdread == -1) + return; + char* bufread = (char*) malloc(10); + int sizeread = gkfs::syscall::gkfs_read(fdread, bufread, 7); - cout << "Reading : " << sizeread << " --> " << bufread << endl; - - gkfs::syscall::gkfs_close(fdread); + cout << "Reading : " << sizeread << " --> " << bufread << endl; + gkfs::syscall::gkfs_close(fdread); } -int main(int argc, char **argv) { - cout << "GekkoFS Client library test" << endl; +int +main(int argc, char** argv) { + cout << "GekkoFS Client library test" << endl; - auto res = gkfs_init(); + auto res = gkfs_init(); - cout << "Init result " << res << endl; + cout << "Init result " << res << endl; - //write_file ("/test.tmp"); - - read_file("/test.tmp"); + write_file("/test.tmp"); - res = gkfs_end(); + read_file("/test.tmp"); - cout << "End result " << res << endl; + res = gkfs_end(); + cout << "End result " << res << endl; } diff --git a/include/client/CMakeLists.txt b/include/client/CMakeLists.txt index 2d12db14e..b82b0b96a 100644 --- a/include/client/CMakeLists.txt +++ b/include/client/CMakeLists.txt @@ -54,3 +54,29 @@ target_sources( syscalls/detail/syscall_info.h ) +target_sources( + gkfs_user_lib + PUBLIC gkfs_functions.hpp + env.hpp + hooks.hpp + intercept.hpp + logging.hpp + make_array.hpp + open_file_map.hpp + open_dir.hpp + path.hpp + preload.hpp + preload_context.hpp + preload_util.hpp + rpc/rpc_types.hpp + rpc/forward_management.hpp + rpc/forward_metadata.hpp + rpc/forward_data.hpp + syscalls/args.hpp + syscalls/decoder.hpp + syscalls/errno.hpp + syscalls/rets.hpp + syscalls/syscall.hpp + syscalls/detail/syscall_info.h + void_syscall_intercept.hpp +) diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 574b084ee..60cbb05a5 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -156,6 +156,7 @@ gkfs_close(unsigned int fd); int gkfs_rename(const std::string& old_path, const std::string& new_path); #endif // HAS_RENAME + } // namespace gkfs::syscall // gkfs_getsingleserverdir is using extern "C" to demangle it for C usage diff --git a/include/client/hooks.hpp b/include/client/hooks.hpp index e71a6a08d..fa9ec424b 100644 --- a/include/client/hooks.hpp +++ b/include/client/hooks.hpp @@ -36,7 +36,13 @@ extern "C" { #include #include } + +#ifndef BYPASS_SYSCALL #include +#else +#include +#endif + /* * For PowerPC, syscall_no_intercept_wrapper() is defined in the diff --git a/include/client/logging.hpp b/include/client/logging.hpp index 2f00058d0..9d1f464e5 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -30,7 +30,11 @@ #ifndef LIBGKFS_LOGGING_HPP #define LIBGKFS_LOGGING_HPP +#ifndef BYPASS_SYSCALL #include +#else +#include +#endif #include #include diff --git a/src/client/void_syscall_intercept.cpp b/include/client/void_syscall_intercept.hpp similarity index 64% rename from src/client/void_syscall_intercept.cpp rename to include/client/void_syscall_intercept.hpp index 744dd7195..16cd01b5b 100644 --- a/src/client/void_syscall_intercept.cpp +++ b/include/client/void_syscall_intercept.hpp @@ -26,24 +26,20 @@ SPDX-License-Identifier: LGPL-3.0-or-later */ +#ifndef SYSCALL_BYPASS_HPP +#define SYSCALL_BYPASS_HPP + +static inline int +syscall_error_code(long result) { + if(result < 0 && result >= -0x1000) + return (int) -result; + + return 0; +} + extern "C" { -int (*intercept_hook_point)(long syscall_number, - long arg0, long arg1, - long arg2, long arg3, - long arg4, long arg5, - long *result) {}; -void (*intercept_hook_point_clone_child)( - unsigned long flags, void *child_stack, - int *ptid, int *ctid, long newtls) {}; -void (*intercept_hook_point_clone_parent)( - unsigned long flags, void *child_stack, - int *ptid, int *ctid, long newtls, - long returned_pid) {}; - void (*intercept_hook_point_post_kernel)(long syscall_number, - long arg0, long arg1, - long arg2, long arg3, - long arg4, long arg5, - long result) {}; - -long syscall_no_intercept(long syscall_number, ...) { return 0}; -} \ No newline at end of file +long +syscall_no_intercept(long syscall_number, ...); +} + +#endif \ No newline at end of file diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 7961f8464..afd514c0b 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -31,14 +31,15 @@ # This builds the `libgkfs_intercept.so` library: the primary GekkoFS client # based on syscall interception. # ############################################################################## -add_library(gkfs_intercept SHARED) + +add_library (gkfs_intercept SHARED) add_library (gkfs_user_lib SHARED) -target_sources( - gkfs_intercept - PRIVATE gkfs_functions.cpp - hooks.cpp + +target_sources(gkfs_intercept + PRIVATE gkfs_functions.cpp intercept.cpp + hooks.cpp logging.cpp open_file_map.cpp open_dir.cpp @@ -50,15 +51,13 @@ target_sources( rpc/forward_data.cpp rpc/forward_management.cpp rpc/forward_metadata.cpp - syscalls/detail/syscall_info.c -) + syscalls/detail/syscall_info.c) target_sources( gkfs_user_lib - PRIVATE void_syscall_intercept.cpp - gkfs_functions.cpp - hooks.cpp + PRIVATE gkfs_functions.cpp intercept.cpp + hooks.cpp logging.cpp open_file_map.cpp open_dir.cpp @@ -70,18 +69,19 @@ target_sources( rpc/forward_data.cpp rpc/forward_management.cpp rpc/forward_metadata.cpp - syscalls/detail/syscall_info.c + syscalls/detail/syscall_info.c syscalls/util.S ) - if(GKFS_ENABLE_AGIOS) - target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) - endif() +target_compile_definitions(gkfs_user_lib PUBLIC BYPASS_SYSCALL) + +if(GKFS_ENABLE_AGIOS) + target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) +endif() target_link_libraries( gkfs_intercept PRIVATE metadata distributor env_util arithmetic path_util rpc_utils - PUBLIC Syscall_intercept::Syscall_intercept - dl + PUBLIC dl Mercury::Mercury hermes fmt::fmt @@ -106,11 +106,9 @@ install( PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) - install( TARGETS gkfs_user_lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) - diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index ef63cd8d0..ea87d44a8 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1396,11 +1396,12 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, /** * @brief Closes an fd. To be used externally - * - * @param fd - * @return int + * + * @param fd + * @return int */ -int gkfs_close(unsigned int fd) { +int +gkfs_close(unsigned int fd) { if(CTX->file_map()->exist(fd)) { // No call to the daemon is required CTX->file_map()->remove(fd); @@ -1412,7 +1413,7 @@ int gkfs_close(unsigned int fd) { // internal fd: ignore it return 0; } - + return -1; } diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index ea7559c46..9a54048c5 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -92,8 +92,8 @@ hook_close(int fd) { LOG(DEBUG, "{}() called with fd: {}", __func__, fd); auto ret = gkfs::syscall::gkfs_close(fd); - - if (ret == 0) + + if(ret == 0) return 0; return syscall_no_intercept_wrapper(SYS_close, fd); diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index 316c5e398..7a8a455fe 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -44,6 +44,24 @@ extern "C" { #include } + +#ifdef BYPASS_SYSCALL +int (*intercept_hook_point)(long syscall_number, long arg0, long arg1, + long arg2, long arg3, long arg4, long arg5, + long* result){}; + +void (*intercept_hook_point_clone_child)(unsigned long flags, void* child_stack, + int* ptid, int* ctid, long newtls){}; + +void (*intercept_hook_point_clone_parent)(unsigned long flags, + void* child_stack, int* ptid, + int* ctid, long newtls, + long returned_pid){}; + +void (*intercept_hook_point_post_kernel)(long syscall_number, long arg0, + long arg1, long arg2, long arg3, + long arg4, long arg5, long result){}; +#endif namespace { thread_local bool reentrance_guard_flag; diff --git a/src/client/path.cpp b/src/client/path.cpp index 4f72b4807..85357209a 100644 --- a/src/client/path.cpp +++ b/src/client/path.cpp @@ -39,9 +39,14 @@ #include #include +#ifndef BYPASS_SYSCALL +#include +#else +#include +#endif + extern "C" { #include -#include } using namespace std; diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 589656796..b989eeb6a 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -67,7 +67,7 @@ exit_error_msg(int errcode, const string& msg) { // if we don't disable interception before calling ::exit() // syscall hooks may find an inconsistent in shared state // (e.g. the logger) and thus, crash - if(CTX->interception_enabled()) { + if(CTX->interception_enabled()) { gkfs::preload::stop_interception(); CTX->disable_interception(); } @@ -316,7 +316,7 @@ destroy_preload() { ld_network_service.reset(); LOG(DEBUG, "RPC subsystem shut down"); - if(CTX->interception_enabled()) { + if(CTX->interception_enabled()) { gkfs::preload::stop_interception(); CTX->disable_interception(); LOG(DEBUG, "Syscall interception stopped"); @@ -326,10 +326,9 @@ destroy_preload() { } - /** * @brief External functions to call linking the library - * + * */ extern "C" int gkfs_init() { @@ -343,7 +342,7 @@ gkfs_init() { extern "C" int gkfs_end() { - CTX->clear_hosts(); + CTX->clear_hosts(); LOG(DEBUG, "Peer information deleted"); ld_network_service.reset(); diff --git a/src/client/preload_context.cpp b/src/client/preload_context.cpp index 7101041dc..5dbc75c87 100644 --- a/src/client/preload_context.cpp +++ b/src/client/preload_context.cpp @@ -37,13 +37,17 @@ #include #include #include - #include #include -extern "C" { +#ifndef BYPASS_SYSCALL #include +#else +#include +#endif + +extern "C" { #include } diff --git a/src/client/syscalls/util.S b/src/client/syscalls/util.S new file mode 100644 index 000000000..67c9b279a --- /dev/null +++ b/src/client/syscalls/util.S @@ -0,0 +1,49 @@ +/* + * Copyright 2016-2017, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +.global syscall_no_intercept; +.type syscall_no_intercept, @function + +.text + +syscall_no_intercept: + movq %rdi, %rax /* convert from linux ABI calling */ + movq %rsi, %rdi /* convention to syscall calling convention */ + movq %rdx, %rsi + movq %rcx, %rdx + movq %r8, %r10 + movq %r9, %r8 + movq 8(%rsp), %r9 + syscall + ret + +.size syscall_no_intercept, .-syscall_no_intercept -- GitLab From 3bd05e270cc885e8f98663f014b05b76aacb7deb Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 11 Oct 2023 11:30:14 +0200 Subject: [PATCH 20/27] Create an user_functions.hpp --- CMakeLists.txt | 2 +- examples/gfind/gkfs_lib_example.cpp | 29 +++++-------- include/client/CMakeLists.txt | 1 + include/client/gkfs_functions.hpp | 1 + include/client/preload.hpp | 8 +--- include/client/user_functions.hpp | 67 +++++++++++++++++++++++++++++ src/client/CMakeLists.txt | 7 +++ src/client/preload.cpp | 8 +++- 8 files changed, 96 insertions(+), 27 deletions(-) create mode 100644 include/client/user_functions.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cda0d59e7..a26d68077 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,7 +38,7 @@ project( enable_testing() #if (NOT CMAKE_COMPILER_IS_GNUCC) - # message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") +# message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") #endif () #if (NOT CMAKE_COMPILER_IS_GNUCXX) # message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported") diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/gfind/gkfs_lib_example.cpp index 513b0ef98..a86dd5812 100644 --- a/examples/gfind/gkfs_lib_example.cpp +++ b/examples/gfind/gkfs_lib_example.cpp @@ -41,35 +41,26 @@ #include #include #include -#include +#include using namespace std; -/* Function exported from GekkoFS LD_PRELOAD, code needs to be compiled with - * -fPIC */ -extern "C" int -gkfs_init() __attribute__((weak)); - -extern "C" int -gkfs_end() __attribute__((weak)); - -void -init_preload(){}; -void -destroy_preload(){}; - void write_file(std::string filename) { // Open File int fd = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDWR | O_CREAT); cout << "FD open " << fd << endl; - char* buf = "testing"; - int size = gkfs::syscall::gkfs_write(fd, buf, 7); + char* bufwrite = (char*) malloc(10); + strncpy(bufwrite, "testing", 7); + - cout << "FD size" << size << endl; + int size = gkfs::syscall::gkfs_write(fd, bufwrite, 7); + cout << "Writting size " << size << endl; + + free(bufwrite); gkfs::syscall::gkfs_close(fd); } @@ -82,10 +73,12 @@ read_file(std::string filename) { char* bufread = (char*) malloc(10); int sizeread = gkfs::syscall::gkfs_read(fdread, bufread, 7); - cout << "Reading : " << sizeread << " --> " << bufread << endl; + cout << "Reading Size: " << sizeread << " Content: " << bufread << endl; + free(bufread); gkfs::syscall::gkfs_close(fdread); } + int main(int argc, char** argv) { cout << "GekkoFS Client library test" << endl; diff --git a/include/client/CMakeLists.txt b/include/client/CMakeLists.txt index b82b0b96a..6815fc953 100644 --- a/include/client/CMakeLists.txt +++ b/include/client/CMakeLists.txt @@ -79,4 +79,5 @@ target_sources( syscalls/syscall.hpp syscalls/detail/syscall_info.h void_syscall_intercept.hpp + user_functions.hpp ) diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 60cbb05a5..6e97df983 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -163,4 +163,5 @@ gkfs_rename(const std::string& old_path, const std::string& new_path); extern "C" int gkfs_getsingleserverdir(const char* path, struct dirent_extended* dirp, unsigned int count, int server); + #endif // GEKKOFS_GKFS_FUNCTIONS_HPP diff --git a/include/client/preload.hpp b/include/client/preload.hpp index 929e30eaf..afad97c68 100644 --- a/include/client/preload.hpp +++ b/include/client/preload.hpp @@ -41,16 +41,12 @@ void init_ld_env_if_needed(); } // namespace gkfs::preload +#ifndef BYPASS_SYSCALL void init_preload() __attribute__((constructor)); void destroy_preload() __attribute__((destructor)); - -extern "C" int -gkfs_init(); - -extern "C" int -gkfs_end(); +#endif #endif // IOINTERCEPT_PRELOAD_HPP diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp new file mode 100644 index 000000000..e041ba61a --- /dev/null +++ b/include/client/user_functions.hpp @@ -0,0 +1,67 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS' POSIX interface. + + GekkoFS' POSIX interface is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GekkoFS' POSIX interface is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with GekkoFS' POSIX interface. If not, see + . + + SPDX-License-Identifier: LGPL-3.0-or-later +*/ + +#ifndef GEKKOFS_USER_FUNCTIONS_HPP +#define GEKKOFS_USER_FUNCTIONS_HPP +#include +#include +#include +#include + +namespace gkfs::syscall { + +int +gkfs_open(const std::string& path, mode_t mode, int flags); + +int +gkfs_remove(const std::string& path); + + +ssize_t +gkfs_write(int fd, const void* buf, size_t count); + + +ssize_t +gkfs_read(int fd, void* buf, size_t count); + + +int +gkfs_close(unsigned int fd); + + +} // namespace gkfs::syscall + + +extern "C" int +gkfs_init(); + +extern "C" int +gkfs_end(); + +#endif // GEKKOFS_USER_FUNCTIONS_HPP diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index afd514c0b..6550147da 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -106,6 +106,13 @@ install( PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) + +set_target_properties(gkfs_user_lib + PROPERTIES + PUBLIC_HEADER "../../include/client/void_syscall_intercept.hpp" +) + + install( TARGETS gkfs_user_lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/src/client/preload.cpp b/src/client/preload.cpp index b989eeb6a..652ef3580 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -333,10 +333,13 @@ destroy_preload() { extern "C" int gkfs_init() { CTX->init_logging(); + // from here ownwards it is safe to print messages LOG(DEBUG, "Logging subsystem initialized"); + gkfs::preload::init_environment(); - return 4; + + return 0; } @@ -349,5 +352,6 @@ gkfs_end() { LOG(DEBUG, "RPC subsystem shut down"); LOG(INFO, "All subsystems shut down. Client shutdown complete."); - return 4; + + return 0; } \ No newline at end of file -- GitLab From 7d2c3868965f58ae5b82d6bf928b5495dcdcb0c4 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 14 Mar 2024 11:53:22 +0100 Subject: [PATCH 21/27] Add headers and new user function headers --- CMake/FindMercury.cmake | 2 +- include/client/user_functions.hpp | 14 +++++++++++--- src/client/CMakeLists.txt | 1 + 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CMake/FindMercury.cmake b/CMake/FindMercury.cmake index 7eb96cde5..451ed6d40 100644 --- a/CMake/FindMercury.cmake +++ b/CMake/FindMercury.cmake @@ -177,7 +177,7 @@ endmacro() include(SelectLibraryConfigurations) -set(_mercury_components na mercury_util mercury_hl) +set(_mercury_components na mercury_util) # prevent repeating work if the main CMakeLists.txt already called # find_package(PkgConfig) diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp index e041ba61a..776e84f3b 100644 --- a/include/client/user_functions.hpp +++ b/include/client/user_functions.hpp @@ -40,20 +40,28 @@ int gkfs_open(const std::string& path, mode_t mode, int flags); int -gkfs_remove(const std::string& path); +gkfs_create(const std::string& path, mode_t mode); +int +gkfs_remove(const std::string& path); ssize_t gkfs_write(int fd, const void* buf, size_t count); - ssize_t gkfs_read(int fd, void* buf, size_t count); - int gkfs_close(unsigned int fd); +off64_t +gkfs_lseek(unsigned int fd, off64_t offset, unsigned int whence); + +ssize_t +gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset); + +ssize_t +gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset); } // namespace gkfs::syscall diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 6550147da..a4827af37 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -110,6 +110,7 @@ install( set_target_properties(gkfs_user_lib PROPERTIES PUBLIC_HEADER "../../include/client/void_syscall_intercept.hpp" + PUBLIC_HEADER "../../include/client/user_functions.hpp" ) -- GitLab From 459f7685b266c6a634eedcf24690282d0e774864 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 28 Feb 2024 17:13:01 +0100 Subject: [PATCH 22/27] Adding gkfs_stat to the user lib --- include/client/user_functions.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp index 776e84f3b..c26e39d43 100644 --- a/include/client/user_functions.hpp +++ b/include/client/user_functions.hpp @@ -34,6 +34,8 @@ #include #include +struct linux_dirent64; + namespace gkfs::syscall { int @@ -63,6 +65,9 @@ gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset); ssize_t gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset); +int +gkfs_stat(const std::string& path, struct stat* buf, bool follow_links = true); + } // namespace gkfs::syscall -- GitLab From 52f21e7ee28637b81a2cc55259e43e277bee6162 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 29 Feb 2024 13:07:10 +0100 Subject: [PATCH 23/27] Add file_list function, remove and expand gkfs_lib example --- examples/gfind/gkfs_lib_example.cpp | 18 ++++++++++++++++++ include/client/gkfs_functions.hpp | 3 +++ include/client/user_functions.hpp | 5 +++++ src/client/gkfs_functions.cpp | 22 ++++++++++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/gfind/gkfs_lib_example.cpp index a86dd5812..4644d29fc 100644 --- a/examples/gfind/gkfs_lib_example.cpp +++ b/examples/gfind/gkfs_lib_example.cpp @@ -91,6 +91,24 @@ main(int argc, char** argv) { read_file("/test.tmp"); + + write_file("/secondfile.tmp"); + + auto f_list = gkfs::syscall::gkfs_get_file_list("/"); + + for(auto f : f_list) { + cout << "File: " << f << endl; + struct stat buf; + memset(&buf, 0, sizeof(struct stat)); + + gkfs::syscall::gkfs_stat("/" + f, &buf, true); + + cout << "Size: " << buf.st_size << " Mode: " << buf.st_mode << endl; + cout << "Atime: " << buf.st_atime << " Mtime: " << buf.st_mtime + << " Ctime: " << buf.st_ctime << endl + << " ****** " << endl; + } + res = gkfs_end(); cout << "End result " << res << endl; diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 6e97df983..313df5fcb 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -152,6 +152,9 @@ gkfs_rmdir(const std::string& path); int gkfs_close(unsigned int fd); +std::vector +gkfs_get_file_list(const std::string& path); + #ifdef HAS_RENAME int gkfs_rename(const std::string& old_path, const std::string& new_path); diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp index c26e39d43..5d1bfdfe4 100644 --- a/include/client/user_functions.hpp +++ b/include/client/user_functions.hpp @@ -68,6 +68,11 @@ gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset); int gkfs_stat(const std::string& path, struct stat* buf, bool follow_links = true); +int +gkfs_remove(const std::string& path); + +std::vector +gkfs_get_file_list(const std::string& path); } // namespace gkfs::syscall diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index ea87d44a8..865d38214 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1504,6 +1504,28 @@ gkfs_readlink(const std::string& path, char* buf, int bufsize) { #endif #endif + +std::vector +gkfs_get_file_list(const std::string& path) { + auto ret = gkfs::rpc::forward_get_dirents(path); + auto err = ret.first; + if(err) { + errno = err; + return {}; + } + + auto open_dir = ret.second; + + std::vector file_list; + unsigned int pos = 0; + + while(pos < open_dir->size()) { + auto de = open_dir->getdent(pos++); + file_list.push_back(de.name()); + } + return file_list; +} + } // namespace gkfs::syscall -- GitLab From a38d0b3dc4ce3573f9581c2ee7e19931996c3ce5 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 14 Mar 2024 12:03:36 +0100 Subject: [PATCH 24/27] Updating all dependencies for v0.9.3 --- scripts/profiles/0.9.3/agios.specs | 16 ++-- scripts/profiles/0.9.3/all.specs | 20 ++--- scripts/profiles/0.9.3/arm.specs | 18 ++--- scripts/profiles/0.9.3/ci.specs | 12 +-- scripts/profiles/0.9.3/default.specs | 18 ++--- scripts/profiles/0.9.3/infiniband_verbs.specs | 16 ++-- scripts/profiles/0.9.3/install/agios.install | 4 +- .../profiles/0.9.3/install/argobots.install | 4 +- .../profiles/0.9.3/install/capstone.install | 4 +- scripts/profiles/0.9.3/install/curl.install | 4 +- scripts/profiles/0.9.3/install/json-c.install | 4 +- .../0.9.3/install/libfabric%verbs.install | 4 +- .../profiles/0.9.3/install/libfabric.install | 4 +- scripts/profiles/0.9.3/install/lz4.install | 4 +- scripts/profiles/0.9.3/install/margo.install | 4 +- .../profiles/0.9.3/install/mercury.install | 4 +- .../profiles/0.9.3/install/parallax.install | 4 +- .../0.9.3/install/prometheus-cpp.install | 4 +- .../profiles/0.9.3/install/rocksdb.install | 4 +- .../0.9.3/install/syscall_intercept.install | 4 +- .../{direct.specs => install/ucx.install} | 81 ++++++++----------- scripts/profiles/0.9.3/marenostrum4.specs | 19 ++--- scripts/profiles/0.9.3/mogon2.specs | 27 ++++--- scripts/profiles/0.9.3/ngio.specs | 19 ++--- scripts/profiles/0.9.3/omnipath_psm2.specs | 19 ++--- scripts/profiles/0.9.3/p9.specs | 19 ++--- scripts/profiles/latest | 2 +- scripts/profiles/sources.list | 3 +- src/daemon/handler/srv_management.cpp | 5 +- tests/scripts/dl_dep.sh/0.8.0/default.out | 2 +- tests/scripts/dl_dep.sh/0.8.0/mogon2.out | 2 +- tests/scripts/dl_dep.sh/0.8.0/ngio.out | 2 +- .../scripts/dl_dep.sh/0.9.0/marenostrum4.out | 2 +- tests/scripts/dl_dep.sh/0.9.0/mogon2.out | 2 +- tests/scripts/dl_dep.sh/0.9.0/ngio.out | 2 +- .../scripts/dl_dep.sh/0.9.0/omnipath_psm2.out | 2 +- tests/scripts/dl_dep.sh/0.9.1/all.out | 2 +- .../scripts/dl_dep.sh/0.9.1/marenostrum4.out | 2 +- tests/scripts/dl_dep.sh/0.9.1/mogon2.out | 2 +- tests/scripts/dl_dep.sh/0.9.1/ngio.out | 2 +- .../scripts/dl_dep.sh/0.9.1/omnipath_psm2.out | 2 +- tests/scripts/dl_dep.sh/0.9.1/p9.out | 2 +- tests/scripts/dl_dep.sh/0.9.2/all.out | 2 +- tests/scripts/dl_dep.sh/0.9.2/arm.out | 2 +- .../scripts/dl_dep.sh/0.9.2/marenostrum4.out | 2 +- tests/scripts/dl_dep.sh/0.9.2/mogon2.out | 2 +- tests/scripts/dl_dep.sh/0.9.2/ngio.out | 2 +- .../scripts/dl_dep.sh/0.9.2/omnipath_psm2.out | 2 +- tests/scripts/dl_dep.sh/0.9.2/p9.out | 2 +- 49 files changed, 192 insertions(+), 198 deletions(-) rename scripts/profiles/0.9.3/{direct.specs => install/ucx.install} (58%) diff --git a/scripts/profiles/0.9.3/agios.specs b/scripts/profiles/0.9.3/agios.specs index 406adb383..68921e298 100644 --- a/scripts/profiles/0.9.3/agios.specs +++ b/scripts/profiles/0.9.3/agios.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,18 +37,18 @@ comment="All dependencies (except transport-specific and experimental)" # Dependencies that must be downloaded directly wgetdeps=( - ["lz4"]="1.9.3" + ["lz4"]="1.9.4" ["capstone"]="4.0.2" ["argobots"]="1.1" - ["rocksdb"]="8.6.7" - ["json-c"]="0.15-20200726" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" ) # Dependencies that must be cloned clonedeps=( - ["libfabric"]="HEAD@v1.13.2" - ["mercury"]="v2.1.0" - ["margo"]="v0.9.6" + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" ) diff --git a/scripts/profiles/0.9.3/all.specs b/scripts/profiles/0.9.3/all.specs index 9780acaa0..67396b8dc 100644 --- a/scripts/profiles/0.9.3/all.specs +++ b/scripts/profiles/0.9.3/all.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,21 +37,21 @@ comment="Dependencies for Mogon 2 supercomputer" # Dependencies that must be downloaded directly wgetdeps=( - ["lz4"]="1.9.3" + ["lz4"]="1.9.4" ["capstone"]="4.0.2" ["argobots"]="1.1" - ["rocksdb"]="8.6.7" + ["rocksdb"]="8.10.0" ["psm2"]="11.2.185" - ["json-c"]="0.15-20200726" - ["curl"]="7.82.0" + ["json-c"]="0.17-20230812" + ["curl"]="7.88.1" ["prometheus-cpp"]="v1.0.0" ) # Dependencies that must be cloned clonedeps=( - ["libfabric"]="HEAD@v1.13.2" - ["mercury"]="v2.1.0" - ["margo"]="v0.9.6" + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" @@ -69,7 +69,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "agios" "curl" "prometheus-cpp" "parallax" ) diff --git a/scripts/profiles/0.9.3/arm.specs b/scripts/profiles/0.9.3/arm.specs index a80ae6214..1d5432de8 100644 --- a/scripts/profiles/0.9.3/arm.specs +++ b/scripts/profiles/0.9.3/arm.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,19 +37,19 @@ comment="Dependencies for PowerPC supercomputer" # Dependencies that must be downloaded directly wgetdeps=( - ["lz4"]="1.9.3" + ["lz4"]="1.9.4" ["capstone"]="4.0.2" ["argobots"]="1.1" - ["rocksdb"]="8.6.7" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" ["psm2"]="11.2.185" - ["json-c"]="0.15-20200726" ) # Dependencies that must be cloned clonedeps=( - ["libfabric"]="HEAD@v1.13.2" - ["mercury"]="v2.1.0" - ["margo"]="v0.9.6" + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" ["syscall_intercept"]="fb73c3924b502e2840a5dc8a18746e395b06a8a6" ) @@ -65,7 +65,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.3/ci.specs b/scripts/profiles/0.9.3/ci.specs index 44c847abc..dc9fcedff 100644 --- a/scripts/profiles/0.9.3/ci.specs +++ b/scripts/profiles/0.9.3/ci.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -38,15 +38,15 @@ comment="Dependencies required by the CI" # Dependencies that must be downloaded directly wgetdeps=( ["argobots"]="1.1" - ["rocksdb"]="8.6.7" + ["rocksdb"]="8.5.4" ["prometheus-cpp"]="v1.0.0" ) # Dependencies that must be cloned clonedeps=( - ["libfabric"]="HEAD@v1.13.2" - ["mercury"]="v2.1.0" - ["margo"]="v0.9.6" + ["libfabric"]="HEAD@v1.17.0" + ["mercury"]="v2.3.0" + ["margo"]="v0.14.1" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ["agios"]="c26a6544200f823ebb8f890dd94e653d148bf226@development" ["parallax"]="ffdea6e820f5c4c2d33e60d9a4b15ef9e6bbcfdd" diff --git a/scripts/profiles/0.9.3/default.specs b/scripts/profiles/0.9.3/default.specs index d214eb0ae..59d8175fe 100644 --- a/scripts/profiles/0.9.3/default.specs +++ b/scripts/profiles/0.9.3/default.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,18 +37,18 @@ comment="All dependencies" # Dependencies that must be downloaded directly wgetdeps=( - ["lz4"]="1.9.3" + ["lz4"]="1.9.4" ["capstone"]="4.0.2" ["argobots"]="1.1" - ["rocksdb"]="8.6.7" - ["json-c"]="0.15-20200726" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" ) -# Dependencies that must be cloned +# Dependencies that must be cloned. clonedeps=( - ["libfabric"]="HEAD@v1.13.2" - ["mercury"]="v2.1.0" - ["margo"]="v0.9.6" + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ) diff --git a/scripts/profiles/0.9.3/infiniband_verbs.specs b/scripts/profiles/0.9.3/infiniband_verbs.specs index 488957246..abb374413 100644 --- a/scripts/profiles/0.9.3/infiniband_verbs.specs +++ b/scripts/profiles/0.9.3/infiniband_verbs.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,18 +37,18 @@ comment="Dependencies for Infiniband supercomputer" # Dependencies that must be downloaded directly wgetdeps=( - ["lz4"]="1.9.3" + ["lz4"]="1.9.4" ["capstone"]="4.0.2" ["argobots"]="1.1" - ["rocksdb"]="8.6.7" - ["json-c"]="0.15-20200726" + ["rocksdb"]="8.5.4" + ["json-c"]="0.17-20230812" ) # Dependencies that must be cloned clonedeps=( - ["libfabric%verbs"]="HEAD@v1.13.2" - ["mercury"]="v2.1.0" - ["margo"]="v0.9.6" + ["libfabric%verbs"]="HEAD@v1.17.0" + ["mercury"]="v2.3.0" + ["margo"]="v0.14.1" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" ) diff --git a/scripts/profiles/0.9.3/install/agios.install b/scripts/profiles/0.9.3/install/agios.install index 3d0392ff5..b5ed7d7ee 100644 --- a/scripts/profiles/0.9.3/install/agios.install +++ b/scripts/profiles/0.9.3/install/agios.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/install/argobots.install b/scripts/profiles/0.9.3/install/argobots.install index 96ccea920..9ff03b5a0 100644 --- a/scripts/profiles/0.9.3/install/argobots.install +++ b/scripts/profiles/0.9.3/install/argobots.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/install/capstone.install b/scripts/profiles/0.9.3/install/capstone.install index ff299d82b..afd7794af 100644 --- a/scripts/profiles/0.9.3/install/capstone.install +++ b/scripts/profiles/0.9.3/install/capstone.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/install/curl.install b/scripts/profiles/0.9.3/install/curl.install index d873819c4..44fab7ce9 100644 --- a/scripts/profiles/0.9.3/install/curl.install +++ b/scripts/profiles/0.9.3/install/curl.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/install/json-c.install b/scripts/profiles/0.9.3/install/json-c.install index 7280cda81..9ba99fed6 100644 --- a/scripts/profiles/0.9.3/install/json-c.install +++ b/scripts/profiles/0.9.3/install/json-c.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/install/libfabric%verbs.install b/scripts/profiles/0.9.3/install/libfabric%verbs.install index 15257eb81..74a192102 100644 --- a/scripts/profiles/0.9.3/install/libfabric%verbs.install +++ b/scripts/profiles/0.9.3/install/libfabric%verbs.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/install/libfabric.install b/scripts/profiles/0.9.3/install/libfabric.install index 7db89438e..6fee0bf9e 100644 --- a/scripts/profiles/0.9.3/install/libfabric.install +++ b/scripts/profiles/0.9.3/install/libfabric.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/install/lz4.install b/scripts/profiles/0.9.3/install/lz4.install index 13e4e53b3..893a1b13b 100644 --- a/scripts/profiles/0.9.3/install/lz4.install +++ b/scripts/profiles/0.9.3/install/lz4.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/install/margo.install b/scripts/profiles/0.9.3/install/margo.install index d2c6e4b5f..014621c68 100644 --- a/scripts/profiles/0.9.3/install/margo.install +++ b/scripts/profiles/0.9.3/install/margo.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/install/mercury.install b/scripts/profiles/0.9.3/install/mercury.install index 601305b83..e2f79ebc0 100644 --- a/scripts/profiles/0.9.3/install/mercury.install +++ b/scripts/profiles/0.9.3/install/mercury.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/install/parallax.install b/scripts/profiles/0.9.3/install/parallax.install index 38bd9c689..8c0cc9e95 100644 --- a/scripts/profiles/0.9.3/install/parallax.install +++ b/scripts/profiles/0.9.3/install/parallax.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/install/prometheus-cpp.install b/scripts/profiles/0.9.3/install/prometheus-cpp.install index 62e420217..154db65df 100644 --- a/scripts/profiles/0.9.3/install/prometheus-cpp.install +++ b/scripts/profiles/0.9.3/install/prometheus-cpp.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/install/rocksdb.install b/scripts/profiles/0.9.3/install/rocksdb.install index 6ece050bd..7c3004fee 100644 --- a/scripts/profiles/0.9.3/install/rocksdb.install +++ b/scripts/profiles/0.9.3/install/rocksdb.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/install/syscall_intercept.install b/scripts/profiles/0.9.3/install/syscall_intercept.install index 5c862dcb7..f69393d3a 100644 --- a/scripts/profiles/0.9.3/install/syscall_intercept.install +++ b/scripts/profiles/0.9.3/install/syscall_intercept.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # diff --git a/scripts/profiles/0.9.3/direct.specs b/scripts/profiles/0.9.3/install/ucx.install similarity index 58% rename from scripts/profiles/0.9.3/direct.specs rename to scripts/profiles/0.9.3/install/ucx.install index 1410254ba..fd90d6d3f 100644 --- a/scripts/profiles/0.9.3/direct.specs +++ b/scripts/profiles/0.9.3/install/ucx.install @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -25,52 +25,37 @@ # # # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ - # vi: ft=bash -# Variables to be imported into the scripts -declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args -declare -a order - -# Comment that should be displayed when printing the profile -comment="Direct dependencies" - -# Dependencies that must be downloaded directly -wgetdeps=( - ["argobots"]="1.1" - ["rocksdb"]="8.6.7" -) - -# Dependencies that must be cloned -clonedeps=( - ["libfabric"]="HEAD@v1.13.2" - ["mercury"]="v2.1.0" - ["margo"]="v0.9.6" - ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" -) - -# Extra arguments for git clone -clonedeps_args=( - ["mercury"]="--recurse-submodules" -) - -# Patches that should be applied post-clone -clonedeps_patches=( - ["syscall_intercept"]="syscall_intercept.patch" -) - -# Ordering that MUST be followed when downloading -order=( - "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" -) +################################################################################ +## The installation script must define both a pkg_install function and +## pkg_check function that, as their name implies, must specify how +## a dependency package should be installed and tested. ## ## The following +## variables can be used in the installation script: +## - CMAKE: a variable that expands to the cmake binary +## - SOURCE_DIR: the directory where the sources for the package were +## downloaded +## - INSTALL_DIR: the directory where the package should be installed +## - CORES: the number of cores to use when building +## - COMPILER_NAME: the name of the compiler being used (e.g. g++, clang, etc.) +## - COMPILER_FULL_VERSION: the compiler's full version (e.g. 9.3.0) +## - COMPILER_MAJOR_VERSION: the compiler's major version (e.g. 9) +## - PERFORM_TEST: whether tests for the package should be executed +################################################################################ -# Extra arguments passed to the installation script. As such, they can -# reference the following variables: -# - CMAKE: a variable that expands to the cmake binary -# - SOURCE_DIR: the directory where the sources for the package were -# downloaded -# - INSTALL_DIR: the directory where the package should be installed -# - CORES: the number of cores to use when building -# - PERFORM_TEST: whether tests for the package should be executed -extra_install_args=( -) +pkg_install() { + + ID="ucx" + CURR="${SOURCE_DIR}/${ID}" + prepare_build_dir "${CURR}" + cd "${CURR}" + ./autogen.sh + cd "${CURR}/build" + ../contrib/configure-release --prefix=${INSTALL_DIR} + make -j"${CORES}" + make install +} + +pkg_check() { + : +} diff --git a/scripts/profiles/0.9.3/marenostrum4.specs b/scripts/profiles/0.9.3/marenostrum4.specs index 7df096a05..15814ec63 100644 --- a/scripts/profiles/0.9.3/marenostrum4.specs +++ b/scripts/profiles/0.9.3/marenostrum4.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,20 +37,21 @@ comment="Dependencies for Marenostrum 4 supercomputer" # Dependencies that must be downloaded directly wgetdeps=( - ["lz4"]="1.9.3" + ["lz4"]="1.9.4" ["capstone"]="4.0.2" ["argobots"]="1.1" - ["rocksdb"]="8.6.7" + ["rocksdb"]="8.5.4" + ["json-c"]="0.17-20230812" ["psm2"]="11.2.185" - ["json-c"]="0.15-20200726" ) # Dependencies that must be cloned clonedeps=( - ["libfabric"]="HEAD@v1.13.2" - ["mercury"]="v2.1.0" - ["margo"]="v0.9.6" + ["libfabric"]="HEAD@v1.17.0" + ["mercury"]="v2.3.0" + ["margo"]="v0.14.1" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) @@ -66,7 +67,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "parallax" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.3/mogon2.specs b/scripts/profiles/0.9.3/mogon2.specs index 0f34f3ff6..f7d9b2cad 100644 --- a/scripts/profiles/0.9.3/mogon2.specs +++ b/scripts/profiles/0.9.3/mogon2.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,21 +37,23 @@ comment="Dependencies for Mogon 2 supercomputer" # Dependencies that must be downloaded directly wgetdeps=( - ["lz4"]="1.9.3" + ["lz4"]="1.9.4" ["capstone"]="4.0.2" ["argobots"]="1.1" - ["rocksdb"]="8.6.7" - ["psm2"]="11.2.185" - ["json-c"]="0.15-20200726" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" +# ["psm2"]="11.2.230" +# ["psm2"]="11.2.185" ) # Dependencies that must be cloned clonedeps=( - ["libfabric"]="HEAD@v1.13.2" - ["mercury"]="v2.1.0" - ["margo"]="v0.9.6" + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" - ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" +# ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) # Extra arguments for git clone @@ -66,7 +68,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "parallax" + "lz4" "capstone" "json-c" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" ) # Extra arguments passed to the installation script. As such, they can @@ -78,5 +80,6 @@ order=( # - CORES: the number of cores to use when building # - PERFORM_TEST: whether tests for the package should be executed extra_install_args=( - ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" + ["libfabric"]="--enable-psm2=yes --enable-opx=yes" +# ["libfabric"]="--enable-psm2=yes --with-psm2-src=${SOURCE_DIR}/psm2" ) diff --git a/scripts/profiles/0.9.3/ngio.specs b/scripts/profiles/0.9.3/ngio.specs index 3eab0bba2..fd1f3a335 100644 --- a/scripts/profiles/0.9.3/ngio.specs +++ b/scripts/profiles/0.9.3/ngio.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,20 +37,21 @@ comment="Dependencies for NEXTGenIO prototype cluster" # Dependencies that must be downloaded directly wgetdeps=( - ["lz4"]="1.9.3" + ["lz4"]="1.9.4" ["capstone"]="4.0.2" ["argobots"]="1.1" - ["rocksdb"]="8.6.7" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" ["psm2"]="11.2.185" - ["json-c"]="0.15-20200726" ) # Dependencies that must be cloned clonedeps=( - ["libfabric"]="HEAD@v1.13.2" - ["mercury"]="v2.1.0" - ["margo"]="v0.9.6" + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" ["parallax"]="c130decd7a71c60c20b98d6a23924f05f754c3cd" ) @@ -66,7 +67,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "parallax" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" "parallax" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.3/omnipath_psm2.specs b/scripts/profiles/0.9.3/omnipath_psm2.specs index 624185f9f..8cdac16d8 100644 --- a/scripts/profiles/0.9.3/omnipath_psm2.specs +++ b/scripts/profiles/0.9.3/omnipath_psm2.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,20 +37,21 @@ comment="Dependencies for Omnipath supercomputer" # Dependencies that must be downloaded directly wgetdeps=( - ["lz4"]="1.9.3" + ["lz4"]="1.9.4" ["capstone"]="4.0.2" ["argobots"]="1.1" - ["rocksdb"]="8.6.7" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" ["psm2"]="11.2.185" - ["json-c"]="0.15-20200726" ) # Dependencies that must be cloned clonedeps=( - ["libfabric"]="HEAD@v1.13.2" - ["mercury"]="v2.1.0" - ["margo"]="v0.9.6" + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" ["syscall_intercept"]="2c8765fa292bc9c28a22624c528580d54658813d" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" ) # Extra arguments for git clone @@ -65,7 +66,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/0.9.3/p9.specs b/scripts/profiles/0.9.3/p9.specs index 6b646f5a3..cf80be5a9 100644 --- a/scripts/profiles/0.9.3/p9.specs +++ b/scripts/profiles/0.9.3/p9.specs @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -37,20 +37,21 @@ comment="Dependencies for PowerPC supercomputer" # Dependencies that must be downloaded directly wgetdeps=( - ["lz4"]="1.9.3" + ["lz4"]="1.9.4" ["capstone"]="4.0.2" ["argobots"]="1.1" - ["rocksdb"]="8.6.7" + ["rocksdb"]="8.10.0" + ["json-c"]="0.17-20230812" ["psm2"]="11.2.185" - ["json-c"]="0.15-20200726" ) # Dependencies that must be cloned clonedeps=( - ["libfabric"]="HEAD@v1.13.2" - ["mercury"]="v2.1.0" - ["margo"]="v0.9.6" + ["libfabric"]="HEAD@v1.20.1" + ["mercury"]="v2.3.1" + ["margo"]="v0.15.0" ["syscall_intercept"]="6eb27a9d2053bb2ac3bb9ce30e13b64ce055c19f" + ["date"]="e7e1482087f58913b80a20b04d5c58d9d6d90155" ) # Extra arguments for git clone @@ -65,7 +66,7 @@ clonedeps_patches=( # Ordering that MUST be followed when downloading order=( - "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" + "lz4" "capstone" "json-c" "psm2" "libfabric" "mercury" "argobots" "margo" "rocksdb" "syscall_intercept" "date" ) # Extra arguments passed to the installation script. As such, they can diff --git a/scripts/profiles/latest b/scripts/profiles/latest index f76f91317..b3ec1638f 120000 --- a/scripts/profiles/latest +++ b/scripts/profiles/latest @@ -1 +1 @@ -0.9.2 \ No newline at end of file +0.9.3 \ No newline at end of file diff --git a/scripts/profiles/sources.list b/scripts/profiles/sources.list index b659f38fb..eec3277d6 100644 --- a/scripts/profiles/sources.list +++ b/scripts/profiles/sources.list @@ -39,7 +39,7 @@ sources=( ["argobots"]="https://github.com/pmodels/argobots/archive/v{{VERSION}}.tar.gz" ["rocksdb"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz" ["rocksdb%experimental"]="https://github.com/facebook/rocksdb/archive/v{{VERSION}}.tar.gz" - ["psm2"]="https://github.com/intel/opa-psm2/archive/PSM2_{{VERSION}}.tar.gz" + ["psm2"]="https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_{{VERSION}}.tar.gz" ["bmi"]="https://github.com/radix-io/bmi/" ["libfabric"]="https://github.com/ofiwg/libfabric.git" ["libfabric%experimental"]="https://github.com/ofiwg/libfabric.git" @@ -53,6 +53,7 @@ sources=( ["parallax"]="https://github.com/CARV-ICS-FORTH/parallax.git" ["prometheus-cpp"]="https://github.com/jupp0r/prometheus-cpp/releases/download/{{VERSION}}/prometheus-cpp-with-submodules.tar.gz" ["curl"]="https://curl.se/download/curl-{{VERSION}}.tar.gz" + ["ucx"]="https://github.com/openucx/ucx/archive/{{VERSION}}.tar.gz" ) diff --git a/src/daemon/handler/srv_management.cpp b/src/daemon/handler/srv_management.cpp index 585646aed..ef5f8735d 100644 --- a/src/daemon/handler/srv_management.cpp +++ b/src/daemon/handler/srv_management.cpp @@ -72,8 +72,9 @@ rpc_srv_get_fs_config(hg_handle_t handle) { out.blocks_state = static_cast(GKFS_DATA->blocks_state()); out.uid = getuid(); out.gid = getgid(); - GKFS_DATA->spdlogger()->debug("{}() Sending output configs back to library", - __func__); + GKFS_DATA->spdlogger()->debug( + "{}() Sending output configs back to library. mountdir '{}' rootdir '{}'", + __func__, out.mountdir, out.rootdir); auto hret = margo_respond(handle, &out); if(hret != HG_SUCCESS) { GKFS_DATA->spdlogger()->error( diff --git a/tests/scripts/dl_dep.sh/0.8.0/default.out b/tests/scripts/dl_dep.sh/0.8.0/default.out index 7ba8e6778..8369fe839 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/default.out +++ b/tests/scripts/dl_dep.sh/0.8.0/default.out @@ -12,5 +12,5 @@ Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocks Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' diff --git a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out index d8af0a79b..d5844433b 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.8.0/mogon2.out @@ -10,4 +10,4 @@ Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.6. Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' diff --git a/tests/scripts/dl_dep.sh/0.8.0/ngio.out b/tests/scripts/dl_dep.sh/0.8.0/ngio.out index f0c9f7959..e6cd7bdcc 100644 --- a/tests/scripts/dl_dep.sh/0.8.0/ngio.out +++ b/tests/scripts/dl_dep.sh/0.8.0/ngio.out @@ -8,6 +8,6 @@ Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa Cloned 'https://github.com/mochi-hpc/mochi-margo' to 'margo' with commit '[v0.6.3]' and flags '' Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' Cloned 'https://github.com/GekkoFS/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' diff --git a/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out index e76b17882..2a2aaf6c2 100644 --- a/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out +++ b/tests/scripts/dl_dep.sh/0.9.0/marenostrum4.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.0/mogon2.out b/tests/scripts/dl_dep.sh/0.9.0/mogon2.out index e76b17882..2a2aaf6c2 100644 --- a/tests/scripts/dl_dep.sh/0.9.0/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.0/mogon2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.0/ngio.out b/tests/scripts/dl_dep.sh/0.9.0/ngio.out index e76b17882..2a2aaf6c2 100644 --- a/tests/scripts/dl_dep.sh/0.9.0/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.0/ngio.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out index e76b17882..2a2aaf6c2 100644 --- a/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.0/omnipath_psm2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/all.out b/tests/scripts/dl_dep.sh/0.9.1/all.out index 35b6b5217..9cbff6342 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/all.out +++ b/tests/scripts/dl_dep.sh/0.9.1/all.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out index 5185b0db8..b90a686dd 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out +++ b/tests/scripts/dl_dep.sh/0.9.1/marenostrum4.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/mogon2.out b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out index 5185b0db8..b90a686dd 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.1/mogon2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/ngio.out b/tests/scripts/dl_dep.sh/0.9.1/ngio.out index 5185b0db8..b90a686dd 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.1/ngio.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out index e76b17882..2a2aaf6c2 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.1/omnipath_psm2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.1/p9.out b/tests/scripts/dl_dep.sh/0.9.1/p9.out index 3a99bb523..b1eee01ba 100644 --- a/tests/scripts/dl_dep.sh/0.9.1/p9.out +++ b/tests/scripts/dl_dep.sh/0.9.1/p9.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/all.out b/tests/scripts/dl_dep.sh/0.9.2/all.out index 4a1ab9e6f..4b5a73fda 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/all.out +++ b/tests/scripts/dl_dep.sh/0.9.2/all.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/arm.out b/tests/scripts/dl_dep.sh/0.9.2/arm.out index 2c61edb9c..8725e5892 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/arm.out +++ b/tests/scripts/dl_dep.sh/0.9.2/arm.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out b/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out index 657154fdc..fe6f4afec 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out +++ b/tests/scripts/dl_dep.sh/0.9.2/marenostrum4.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/mogon2.out b/tests/scripts/dl_dep.sh/0.9.2/mogon2.out index 657154fdc..fe6f4afec 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/mogon2.out +++ b/tests/scripts/dl_dep.sh/0.9.2/mogon2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/ngio.out b/tests/scripts/dl_dep.sh/0.9.2/ngio.out index 657154fdc..fe6f4afec 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/ngio.out +++ b/tests/scripts/dl_dep.sh/0.9.2/ngio.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out b/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out index d26b498ca..96e22237b 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out +++ b/tests/scripts/dl_dep.sh/0.9.2/omnipath_psm2.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' diff --git a/tests/scripts/dl_dep.sh/0.9.2/p9.out b/tests/scripts/dl_dep.sh/0.9.2/p9.out index 3a99bb523..b1eee01ba 100644 --- a/tests/scripts/dl_dep.sh/0.9.2/p9.out +++ b/tests/scripts/dl_dep.sh/0.9.2/p9.out @@ -1,7 +1,7 @@ Downloaded 'https://github.com/lz4/lz4/archive/v1.9.3.tar.gz' to 'lz4' Downloaded 'https://github.com/aquynh/capstone/archive/4.0.2.tar.gz' to 'capstone' Downloaded 'https://github.com/json-c/json-c/archive/json-c-0.15-20200726.tar.gz' to 'json-c' -Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' +Downloaded 'https://github.com/cornelisnetworks/opa-psm2/archive/PSM2_11.2.185.tar.gz' to 'psm2' Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.13.2' Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[v2.1.0]' and flags '--recurse-submodules' Downloaded 'https://github.com/pmodels/argobots/archive/v1.1.tar.gz' to 'argobots' -- GitLab From d7029f2b3b26d3ebfbb1daa66152bb3f10e01fa6 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Tue, 27 Jun 2023 18:51:02 +0200 Subject: [PATCH 25/27] Fix margo deprecated code --- src/daemon/handler/srv_data.cpp | 4 ++-- src/daemon/handler/srv_metadata.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index 9a4e16973..d28d8ba2c 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -109,7 +109,7 @@ rpc_srv_write(hg_handle_t handle) { return gkfs::rpc::cleanup_respond(&handle, &in, &out, &bulk_handle); } auto hgi = margo_get_info(handle); - auto mid = margo_hg_info_get_instance(hgi); + auto mid = margo_hg_handle_get_instance(handle); auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug( "{}() path: '{}' chunk_start '{}' chunk_end '{}' chunk_n '{}' total_chunk_size '{}' bulk_size: '{}' offset: '{}'", @@ -414,7 +414,7 @@ rpc_srv_read(hg_handle_t handle) { return gkfs::rpc::cleanup_respond(&handle, &in, &out, &bulk_handle); } auto hgi = margo_get_info(handle); - auto mid = margo_hg_info_get_instance(hgi); + auto mid = margo_hg_handle_get_instance(handle); auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug( diff --git a/src/daemon/handler/srv_metadata.cpp b/src/daemon/handler/srv_metadata.cpp index e68069797..1339170b9 100644 --- a/src/daemon/handler/srv_metadata.cpp +++ b/src/daemon/handler/srv_metadata.cpp @@ -537,7 +537,7 @@ rpc_srv_get_dirents(hg_handle_t handle) { // Retrieve size of source buffer auto hgi = margo_get_info(handle); - auto mid = margo_hg_info_get_instance(hgi); + auto mid = margo_hg_handle_get_instance(handle); auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug("{}() Got RPC: path '{}' bulk_size '{}' ", __func__, in.path, bulk_size); @@ -696,7 +696,7 @@ rpc_srv_get_dirents_extended(hg_handle_t handle) { // Retrieve size of source buffer auto hgi = margo_get_info(handle); - auto mid = margo_hg_info_get_instance(hgi); + auto mid = margo_hg_handle_get_instance(handle); auto bulk_size = margo_bulk_get_size(in.bulk_handle); GKFS_DATA->spdlogger()->debug("{}() Got RPC: path '{}' bulk_size '{}' ", __func__, in.path, bulk_size); -- GitLab From f841554f5cef04712d7bc8bb1f9d598088a57390 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Tue, 27 Jun 2023 18:52:53 +0200 Subject: [PATCH 26/27] Migrated to margo-hg-shim --- include/client/rpc/rpc_types.hpp | 64 ++++++++++++++++---------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/include/client/rpc/rpc_types.hpp b/include/client/rpc/rpc_types.hpp index a3fa9b64a..4c756dec2 100644 --- a/include/client/rpc/rpc_types.hpp +++ b/include/client/rpc/rpc_types.hpp @@ -85,10 +85,10 @@ struct fs_config { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 3033006080; + constexpr static const uint64_t public_id = 1; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::fs_config; @@ -259,10 +259,10 @@ struct create { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 796590080; + constexpr static const uint64_t public_id = 2; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::create; @@ -375,10 +375,10 @@ struct stat { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 1396244480; + constexpr static const uint64_t public_id = 3; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::stat; @@ -494,10 +494,10 @@ struct remove_metadata { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 2087845888; + constexpr static const uint64_t public_id = 4; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::remove_metadata; @@ -618,10 +618,10 @@ struct decr_size { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 1291649024; + constexpr static const uint64_t public_id = 5; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::decr_size; @@ -734,10 +734,10 @@ struct update_metadentry { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 99483648; + constexpr static const uint64_t public_id = 6; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::update_metadentry; @@ -960,10 +960,10 @@ struct get_metadentry_size { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 3426484224; + constexpr static const uint64_t public_id = 7; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::get_metadentry_size; @@ -1076,10 +1076,10 @@ struct update_metadentry_size { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 2760900608; + constexpr static const uint64_t public_id = 8; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::update_metadentry_size; @@ -1216,10 +1216,10 @@ struct mk_symlink { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 3207004160; + constexpr static const uint64_t public_id = 9; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::mk_symlink; @@ -1334,10 +1334,10 @@ struct remove_data { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 2649292800; + constexpr static const uint64_t public_id = 10; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::remove_data; @@ -1442,10 +1442,10 @@ struct write_data { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 3716481024; + constexpr static const uint64_t public_id = 11; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::write; @@ -1627,10 +1627,10 @@ struct read_data { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 892207104; + constexpr static const uint64_t public_id = 12; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::read; @@ -1812,10 +1812,10 @@ struct trunc_data { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 1850933248; + constexpr static const uint64_t public_id = 13; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::truncate; @@ -1931,10 +1931,10 @@ struct get_dirents { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 4121034752; + constexpr static const uint64_t public_id = 14; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::get_dirents; @@ -2055,10 +2055,10 @@ struct get_dirents_extended { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 1463943168; + constexpr static const uint64_t public_id = 15; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::get_dirents_extended; @@ -2181,10 +2181,10 @@ struct chunk_stat { // RPC public identifier // (N.B: we reuse the same IDs assigned by Margo so that the daemon // understands Hermes RPCs) - constexpr static const uint64_t public_id = 532742144; + constexpr static const uint64_t public_id = 16; // RPC internal Mercury identifier - constexpr static const hg_id_t mercury_id = public_id; + constexpr static const hg_id_t mercury_id = 0; // RPC name constexpr static const auto name = gkfs::rpc::tag::get_chunk_stat; -- GitLab From 1c49dbc1733c57e6379613de496801274b630d46 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 14 Mar 2024 16:38:50 +0100 Subject: [PATCH 27/27] Set Hermes submodule to master --- external/hermes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/hermes b/external/hermes index d8755039c..a133c9dd0 160000 --- a/external/hermes +++ b/external/hermes @@ -1 +1 @@ -Subproject commit d8755039c82501323cdc9a4f5a2b922b9b3bcac7 +Subproject commit a133c9dd0cefc7e8a3554badfb8995c9f2c27be6 -- GitLab