diff --git a/CMake/FindMercury.cmake b/CMake/FindMercury.cmake index fd83231c1095dda367a3220de15324c1b7bcff54..73d8f435c4a67b3a735bffe259319cc5ab5c8069 100644 --- a/CMake/FindMercury.cmake +++ b/CMake/FindMercury.cmake @@ -177,7 +177,7 @@ endmacro() include(SelectLibraryConfigurations) -set(_mercury_components na mchecksum mercury_util mercury_hl) +set(_mercury_components na mercury_util mercury_hl) # prevent repeating work if the main CMakeLists.txt already called # find_package(PkgConfig) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e701f757d39858edb1c8da2eb476fd9b4ad4d4f..71e7c842f6c73bc814feaff9203b6c97048f8058 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -174,7 +174,7 @@ endif() if(GKFS_ENABLE_ROCKSDB) message(STATUS "[${PROJECT_NAME}] Checking for RocksDB") add_compile_definitions(GKFS_ENABLE_ROCKSDB) - find_package(RocksDB 6.26.1 REQUIRED) + find_package(RocksDB REQUIRED) message(STATUS "[${PROJECT_NAME}] RocksDB version ${RocksDB_VERSION}") message(STATUS "[${PROJECT_NAME}] RocksDB location ${RocksDB_DIR}") endif() diff --git a/CMakePresets.json b/CMakePresets.json index d222b830a073edf13642bb626e6b089b453a1656..88fc9ce80df87c14a5217b3334b7e30e1dd8bfcb 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -79,12 +79,14 @@ "cacheVariables": { "CMAKE_INSTALL_PREFIX": "${sourceDir}/gkfs/install", "GKFS_USE_GUIDED_DISTRIBUTION": true, - "GKFS_ENABLE_PARALLAX": true, + "GKFS_ENABLE_PARALLAX": false, "GKFS_BUILD_TESTS": true, "GKFS_INSTALL_TESTS": true, "GKFS_CHUNK_STATS": true, "GKFS_ENABLE_PROMETHEUS": true, - "GKFS_RENAME_SUPPORT": true + "GKFS_RENAME_SUPPORT": true, + "MAX_OPEN_FDS": "10000", + "MAX_INTERNAL_FDS": "1024" } }, { diff --git a/examples/gfind/gfind.cpp b/examples/gfind/gfind.cpp index 59b3b76086798419ceb4984628072263f45721c1..e00b156077e6906ce10368001b045f742dbafc86 100644 --- a/examples/gfind/gfind.cpp +++ b/examples/gfind/gfind.cpp @@ -42,6 +42,7 @@ #include #include #include +#include using namespace std; @@ -114,7 +115,7 @@ static void pfind_print_help(pfind_options_t *res) { "\tworkdir = \"%s\"\n" "\t-newer = \"%s\"\n" "\t-name|-regex = \"%s\"\n" - "\t-S: num servers = \"%s\"\n" + "\t-S: num servers = \"%d\"\n" "\t-M: mountdir = \"%s\"\n" "Optional flags\n" "\t-h: prints the help\n" diff --git a/examples/gfind/sfind.cpp b/examples/gfind/sfind.cpp index 92f25c2f86613a65ddf8a5b4d1278c04d55c31d9..319ceac994aef344a50833ea5c4099ae07defe65 100644 --- a/examples/gfind/sfind.cpp +++ b/examples/gfind/sfind.cpp @@ -41,6 +41,7 @@ #include #include #include +#include using namespace std; diff --git a/include/client/open_file_map.hpp b/include/client/open_file_map.hpp index cc67913b5f4ed66cc0b7be96338cab9dc26714ad..50963d68a4f4d0f0362ef5766302b470f6da5e74 100644 --- a/include/client/open_file_map.hpp +++ b/include/client/open_file_map.hpp @@ -35,6 +35,7 @@ #include #include #include +#include namespace gkfs::filemap { diff --git a/include/common/metadata.hpp b/include/common/metadata.hpp index 1ae0f4046668716ef9320693d8fd62f72850e2ff..d64f94921439be0d738c6dda01b7059f94457dbf 100644 --- a/include/common/metadata.hpp +++ b/include/common/metadata.hpp @@ -35,6 +35,8 @@ #include #include #include +#include + namespace gkfs::metadata { diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index 3425ed300b5675fa3ed5b1eee0a5586771cb0966..de552de67031339fcfc91269f635819c28a4c22b 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -26,7 +26,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ -import os, sh, sys, re, pytest, signal +import os, sh, shutil, sys, re, pytest, signal import random, socket, netifaces, time from pathlib import Path from itertools import islice @@ -149,13 +149,11 @@ def _process_exists(pid): """ try: - sh.ps(['-h', '-p', pid]) - except Exception: - # sh.raises an Exception if the command doesn't return 0 + os.kill(pid, 0) + return True + except OSError: return False - return True - def _find_search_paths(additional_paths=None): """ Return the entire list of search paths available to the process. If @@ -240,8 +238,8 @@ class Daemon: self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, - 'GKFS_HOSTS_FILE' : self.cwd / gkfs_hosts_file, - 'GKFS_DAEMON_LOG_PATH' : self.logdir / gkfs_daemon_log_file, + 'GKFS_HOSTS_FILE' : str(self.cwd / gkfs_hosts_file), + 'GKFS_DAEMON_LOG_PATH' : str(self.logdir / gkfs_daemon_log_file), 'GKFS_DAEMON_LOG_LEVEL': gkfs_daemon_log_level, } self._env.update(self._patched_env) @@ -291,6 +289,8 @@ class Daemon: return self + + def wait_until_active(self, pid, timeout, max_lines=50): """ Waits until a GKFS daemon is active or until a certain timeout @@ -325,7 +325,7 @@ class Daemon: logger.debug(f"daemon log file missing, checking if daemon is alive...") pid=self._proc.pid - + if not _process_exists(pid): raise RuntimeError(f"process {pid} is not running") @@ -416,10 +416,10 @@ class Client: self._patched_env = { 'LD_LIBRARY_PATH': libdirs, - 'LD_PRELOAD': self._preload_library, - 'LIBGKFS_HOSTS_FILE': self.cwd / gkfs_hosts_file, + 'LD_PRELOAD': str(self._preload_library), + 'LIBGKFS_HOSTS_FILE': str(self.cwd / gkfs_hosts_file), 'LIBGKFS_LOG': gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT': self._workspace.logdir / gkfs_client_log_file, + 'LIBGKFS_LOG_OUTPUT': str(self._workspace.logdir / gkfs_client_log_file), 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } @@ -517,10 +517,10 @@ class ShellClient: self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, - 'LD_PRELOAD' : self._preload_library, - 'LIBGKFS_HOSTS_FILE' : self.cwd / gkfs_hosts_file, + 'LD_PRELOAD' : str(self._preload_library), + 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfs_hosts_file), 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : self._workspace.logdir / gkfs_client_log_file + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file) } self._env.update(self._patched_env) @@ -661,7 +661,11 @@ class ShellClient: extra properties to it. """ - found_cmd = sh.which(cmd, self._search_paths) + + + found_cmd = shutil.which(cmd, + path=':'.join(str(p) for p in self._search_paths) + ) if not found_cmd: raise sh.CommandNotFound(cmd) @@ -717,8 +721,8 @@ class FwdDaemon: self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, - 'GKFS_HOSTS_FILE' : self.cwd / gkfwd_hosts_file, - 'GKFS_DAEMON_LOG_PATH' : self.logdir / gkfwd_daemon_log_file, + 'GKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), + 'GKFS_DAEMON_LOG_PATH' : str(self.logdir / gkfwd_daemon_log_file), 'GKFS_DAEMON_LOG_LEVEL': gkfwd_daemon_log_level } self._env.update(self._patched_env) @@ -899,10 +903,10 @@ class FwdClient: self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, 'LD_PRELOAD' : self._preload_library, - 'LIBGKFS_HOSTS_FILE' : self.cwd / gkfwd_hosts_file, - 'LIBGKFS_FORWARDING_MAP_FILE' : self.cwd / gkfwd_forwarding_map_file_local, + 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), + 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file_local), 'LIBGKFS_LOG' : gkfs_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : self._workspace.logdir / gkfwd_client_log_file_local + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file_local) } self._env.update(self._patched_env) @@ -988,11 +992,11 @@ class ShellFwdClient: self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, - 'LD_PRELOAD' : self._preload_library, - 'LIBGKFS_HOSTS_FILE' : self.cwd / gkfwd_hosts_file, - 'LIBGKFS_FORWARDING_MAP_FILE' : self.cwd / gkfwd_forwarding_map_file, + 'LD_PRELOAD' : str(self._preload_library), + 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), + 'LIBGKFS_FORWARDING_MAP_FILE' : str(self.cwd / gkfwd_forwarding_map_file), 'LIBGKFS_LOG' : gkfwd_client_log_level, - 'LIBGKFS_LOG_OUTPUT' : self._workspace.logdir / gkfwd_client_log_file + 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfwd_client_log_file) } self._env.update(self._patched_env) diff --git a/tests/integration/requirements.txt.in b/tests/integration/requirements.txt.in index 1d4d1a8f334350c6ad8505f122c3b390c2ca51b4..a58d2b89be9cf9ce7d57d7ac0bcc224c92fad6ce 100644 --- a/tests/integration/requirements.txt.in +++ b/tests/integration/requirements.txt.in @@ -27,7 +27,7 @@ pytest==6.2.5 pytest-dependency==0.5.1 pytest-forked==1.1.3 pytest-xdist==1.31.0 -sh==1.12.14 +sh==1.14.3 six==1.14.0 toml==0.10.2 traitlets==4.3.3