Loading tests/integration/conftest.py +20 −1 Original line number Diff line number Diff line Loading @@ -34,7 +34,7 @@ from pathlib import Path from harness.logger import logger, initialize_logging, finalize_logging from harness.cli import add_cli_options, set_default_log_formatter from harness.workspace import Workspace, FileCreator from harness.gkfs import Daemon, Client, Proxy, ShellClient, FwdDaemon, FwdClient, ShellFwdClient, FwdDaemonCreator, FwdClientCreator from harness.gkfs import Daemon, Client, ClientLibc, Proxy, ShellClient, ShellClientLibc, FwdDaemon, FwdClient, ShellFwdClient, FwdDaemonCreator, FwdClientCreator from harness.reporter import report_test_status, report_test_headline, report_assertion_pass def pytest_configure(config): Loading Loading @@ -159,6 +159,16 @@ def gkfs_client_proxy(test_workspace): return Client(test_workspace, True) @pytest.fixture def gkfs_clientLibc(test_workspace): """ Sets up a gekkofs client environment so that operations (system calls, library calls, ...) can be requested from a co-running daemon. """ return ClientLibc(test_workspace) @pytest.fixture def gkfs_shell(test_workspace): """ Loading @@ -177,6 +187,15 @@ def gkfs_shell_proxy(test_workspace): return ShellClient(test_workspace,True) @pytest.fixture def gkfs_shellLibc(test_workspace): """ Sets up a gekkofs environment so that shell commands (stat, ls, mkdir, etc.) can be issued to a co-running daemon. """ return ShellClientLibc(test_workspace) @pytest.fixture def file_factory(test_workspace): """ Loading tests/integration/conftest.template +20 −1 Original line number Diff line number Diff line Loading @@ -34,7 +34,7 @@ from pathlib import Path from harness.logger import logger, initialize_logging, finalize_logging from harness.cli import add_cli_options, set_default_log_formatter from harness.workspace import Workspace, FileCreator from harness.gkfs import Daemon, Client, Proxy, ShellClient, FwdDaemon, FwdClient, ShellFwdClient, FwdDaemonCreator, FwdClientCreator from harness.gkfs import Daemon, Client, ClientLibc, Proxy, ShellClient, ShellClientLibc, FwdDaemon, FwdClient, ShellFwdClient, FwdDaemonCreator, FwdClientCreator from harness.reporter import report_test_status, report_test_headline, report_assertion_pass def pytest_configure(config): Loading Loading @@ -159,6 +159,16 @@ def gkfs_client_proxy(test_workspace): return Client(test_workspace, True) @pytest.fixture def gkfs_clientLibc(test_workspace): """ Sets up a gekkofs client environment so that operations (system calls, library calls, ...) can be requested from a co-running daemon. """ return ClientLibc(test_workspace) @pytest.fixture def gkfs_shell(test_workspace): """ Loading @@ -177,6 +187,15 @@ def gkfs_shell_proxy(test_workspace): return ShellClient(test_workspace,True) @pytest.fixture def gkfs_shellLibc(test_workspace): """ Sets up a gekkofs environment so that shell commands (stat, ls, mkdir, etc.) can be issued to a co-running daemon. """ return ShellClientLibc(test_workspace) @pytest.fixture def file_factory(test_workspace): """ Loading tests/integration/harness/gkfs.py +312 −3 Original line number Diff line number Diff line Loading @@ -40,6 +40,7 @@ from harness.cmd import CommandParser gkfs_daemon_cmd = 'gkfs_daemon' gkfs_client_cmd = 'gkfs.io' gkfs_client_lib_file = 'libgkfs_intercept.so' gkfs_client_lib_libc_file = 'libgkfs_libc_intercept.so' gkfs_hosts_file = 'gkfs_hosts.txt' gkfs_daemon_log_file = 'gkfs_daemon.log' gkfs_daemon_log_level = '100' Loading Loading @@ -255,10 +256,10 @@ class Daemon: def run(self): args = ['--mountdir', self.mountdir, '--rootdir', self.rootdir, args = ['--mountdir', self.mountdir.as_posix(), '--rootdir', self.rootdir.as_posix(), '-l', self._address, '--metadir', self._metadir, '--metadir', self._metadir.as_posix(), '--dbbackend', self._database, '--output-stats', self.logdir / 'stats.log', '--enable-collection', Loading Loading @@ -602,6 +603,87 @@ class Client: def cwd(self): return self._workspace.twd class ClientLibc: """ A class to represent a GekkoFS client process with a patched LD_PRELOAD. This class allows tests to interact with the file system using I/O-related function calls, be them system calls (e.g. read()) or glibc I/O functions (e.g. opendir()). """ def __init__(self, workspace): self._parser = IOParser() self._workspace = workspace self._cmd = sh.Command(gkfs_client_cmd, self._workspace.bindirs) self._env = os.environ.copy() libdirs = ':'.join( filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + [str(p) for p in self._workspace.libdirs])) # ensure the client interception library is available: # to avoid running code with potentially installed libraries, # it must be found in one (and only one) of the workspace's bindirs preloads = [] for d in self._workspace.bindirs: search_path = Path(d) / gkfs_client_lib_libc_file if search_path.exists(): preloads.append(search_path) if len(preloads) == 0: logger.error(f'No client libraries found in the test\'s binary directories:') pytest.exit("Aborted due to initialization error. Check test logs.") if len(preloads) != 1: logger.error(f'Multiple client libraries found in the test\'s binary directories:') for p in preloads: logger.error(f' {p}') logger.error(f'Make sure that only one copy of the client library is available.') pytest.exit("Aborted due to initialization error. Check test logs.") self._preload_library = preloads[0] self._patched_env = { 'LD_LIBRARY_PATH': libdirs, 'LD_PRELOAD': str(self._preload_library), 'LIBGKFS_HOSTS_FILE': str(self.cwd / gkfs_hosts_file), 'LIBGKFS_LOG': gkfs_client_log_level, 'LIBGKFS_LOG_OUTPUT': str(self._workspace.logdir / gkfs_client_log_file), 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @property def preload_library(self): """ Return the preload library detected for this client """ return self._preload_library def run(self, cmd, *args): logger.debug(f"running client") logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, list(args)))) logger.debug(f"patched env: {pformat(self._patched_env)}") out = self._cmd( [ cmd ] + list(args), _env = self._env, # _out=sys.stdout, # _err=sys.stderr, ) logger.debug(f"command output: {out.stdout}") return self._parser.parse(cmd, out.stdout) def __getattr__(self, name): return _proxy_exec(self, name) @property def cwd(self): return self._workspace.twd class ShellCommand: """ A wrapper class for sh.RunningCommand that allows seamlessly using all Loading Loading @@ -814,6 +896,233 @@ class ShellClient: found_cmd = shutil.which(cmd, path=':'.join(str(p) for p in self._search_paths) ) if not found_cmd: raise sh.CommandNotFound(cmd) self._cmd = sh.Command(found_cmd) logger.debug(f"running program") logger.debug(f"cmd: {cmd} {' '.join(str(a) for a in args)}") logger.debug(f"search_paths: {':'.join(str(p) for p in self._search_paths)}") logger.debug(f"timeout: {timeout} seconds") logger.debug(f"timeout_signal: {signal.Signals(timeout_signal).name}") logger.debug(f"patched env:\n{pformat(self._patched_env)}") # 'sh' raises an exception if the return code is not zero; # since we'd rather check for return codes explictly, we # whitelist all exit codes from 1 to 255 as 'ok' using the # _ok_code argument proc = self._cmd( args, _env = self._env, # _out=sys.stdout, # _err=sys.stderr, _timeout=timeout, _timeout_signal=timeout_signal, # _ok_code=list(range(0, 256)) ) logger.debug(f"program stdout: {proc.stdout}") logger.debug(f"program stderr: {proc.stderr}") return ShellCommand(cmd, proc) def __getattr__(self, name): return _proxy_exec(self, name) @property def cwd(self): return self._workspace.twd class ShellClientLibc: """ A class to represent a GekkoFS shell client process. This class allows tests to execute shell commands or scripts via bash -c on a GekkoFS instance. """ def __init__(self, workspace): self._workspace = workspace self._search_paths = _find_search_paths(self._workspace.bindirs) self._env = os.environ.copy() libdirs = ':'.join( filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + [str(p) for p in self._workspace.libdirs])) # ensure the client interception library is available: # to avoid running code with potentially installed libraries, # it must be found in one (and only one) of the workspace's bindirs preloads = [] for d in self._workspace.bindirs: search_path = Path(d) / gkfs_client_lib_libc_file if search_path.exists(): preloads.append(search_path) if len(preloads) != 1: logger.error(f'Multiple client libraries found in the test\'s binary directories:') for p in preloads: logger.error(f' {p}') logger.error(f'Make sure that only one copy of the client library is available.') pytest.exit("Aborted due to initialization error") self._preload_library = preloads[0] self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, 'LD_PRELOAD' : str(self._preload_library), 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfs_hosts_file), 'LIBGKFS_LOG' : gkfs_client_log_level, 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file), 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @property def patched_environ(self): """ Return the patched environment required to run a test as a string that can be prepended to a shell command. """ return ' '.join(f'{k}="{v}"' for k,v in self._patched_env.items()) def script(self, code, intercept_shell=True, timeout=60, timeout_signal=signal.SIGKILL): """ Execute a shell script passed as an argument in bash. For instance, the following snippet: mountdir = pathlib.Path('/tmp') file01 = 'file01' ShellClient().script( f''' expected_pathname={mountdir / file01} if [[ -e ${{expected_pathname}} ]]; then exit 0 fi exit 1 ''') transforms into: bash -c ' expected_pathname=/tmp/file01 if [[ -e ${expected_pathname} ]]; then exit 0 fi exit 1 ' Note that since we are using Python's f-strings, for variable expansions to work correctly, they need to be defined with double braces, e.g. ${{expected_pathname}}. Parameters ---------- code: `str` The script code to be passed to 'bash -c'. intercept_shell: `bool` Controls whether the shell executing the script should be executed with LD_PRELOAD=libgkfs_intercept.so (default: True). timeout: `int` How much time, in seconds, we should give the process to complete. If the process does not finish within the timeout, it will be sent the signal defined by `timeout_signal`. Default value: 60 timeout_signal: `int` The signal to be sent to the process if `timeout` is not None. Default value: signal.SIGKILL Returns ------- A sh.RunningCommand instance that allows interacting with the finished process. """ logger.debug(f"running bash") logger.debug(f"cmd: bash -c '{code}'") logger.debug(f"timeout: {timeout} seconds") logger.debug(f"timeout_signal: {signal.Signals(timeout_signal).name}") if intercept_shell: logger.debug(f"patched env: {self._patched_env}") self._cmd = sh.Command("bash") # 'sh' raises an exception if the return code is not zero; # since we'd rather check for return codes explictly, we # whitelist all exit codes from 1 to 255 as 'ok' using the # _ok_code argument return self._cmd('-c', code, _env = (self._env if intercept_shell else os.environ), # _out=sys.stdout, # _err=sys.stderr, _timeout=timeout, _timeout_signal=timeout_signal, # _ok_code=list(range(0, 256)) ) def run(self, cmd, *args, timeout=60, timeout_signal=signal.SIGKILL): """ Execute a shell command with arguments. For example, the following snippet: mountdir = pathlib.Path('/tmp') file01 = 'file01' ShellClient().stat('--terse', mountdir / file01) transforms into: bash -c 'stat --terse /tmp/file01' Parameters: ----------- cmd: `str` The command to execute. args: `list` The list of arguments for the command. timeout: `number` How much time, in seconds, we should give the process to complete. If the process does not finish within the timeout, it will be sent the signal defined by `timeout_signal`. Default value: 60 timeout_signal: `int` The signal to be sent to the process if `timeout` is not None. Default value: signal.SIGKILL Returns ------- A ShellCommand instance that allows interacting with the finished process. Note that ShellCommand wraps sh.RunningCommand and adds s extra properties to it. """ found_cmd = shutil.which(cmd, path=':'.join(str(p) for p in self._search_paths) ) Loading tests/integration/operations/test_read_operations.py +128 −0 Original line number Diff line number Diff line Loading @@ -192,3 +192,131 @@ def test_preadv(gkfs_daemon, gkfs_client): assert ret.buf_0 == buf_0 assert ret.buf_1 == buf_1 assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes def test_read_libc(gkfs_daemon, gkfs_clientLibc): file = gkfs_daemon.mountdir / "file" # create a file in gekkofs ret = gkfs_clientLibc.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # write a buffer we know buf = b'42' ret = gkfs_clientLibc.write(file, buf, len(buf)) assert ret.retval == len(buf) # Return the number of written bytes # open the file to read ret = gkfs_clientLibc.open(file, os.O_RDONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # read the file ret = gkfs_clientLibc.read(file, len(buf)) assert ret.buf == buf assert ret.retval == len(buf) # Return the number of read bytes def test_pread_libc(gkfs_daemon, gkfs_clientLibc): file = gkfs_daemon.mountdir / "file" # create a file in gekkofs ret = gkfs_clientLibc.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # write a buffer we know buf = b'42' ret = gkfs_clientLibc.pwrite(file, buf, len(buf), 1024) assert ret.retval == len(buf) # Return the number of written bytes # open the file to read ret = gkfs_clientLibc.open(file, os.O_RDONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # read the file at offset 1024 ret = gkfs_clientLibc.pread(file, len(buf), 1024) assert ret.buf == buf assert ret.retval == len(buf) # Return the number of read bytes @pytest.mark.skip(reason="readv not implemented in libc") def test_readv_libc(gkfs_daemon, gkfs_clientLibc): file = gkfs_daemon.mountdir / "file" # create a file in gekkofs ret = gkfs_clientLibc.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # write a buffer we know buf_0 = b'42' buf_1 = b'24' ret = gkfs_clientLibc.writev(file, buf_0, buf_1, 2) assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes # open the file to read ret = gkfs_clientLibc.open(file, os.O_RDONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # read the file ret = gkfs_clientLibc.readv(file, len(buf_0), len(buf_1)) assert ret.buf_0 == buf_0 assert ret.buf_1 == buf_1 assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes @pytest.mark.skip(reason="preadv not implemented in libc") def test_preadv_libc(gkfs_daemon, gkfs_clientLibc): file = gkfs_daemon.mountdir / "file" # create a file in gekkofs ret = gkfs_clientLibc.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # write a buffer we know buf_0 = b'42' buf_1 = b'24' ret = gkfs_clientLibc.pwritev(file, buf_0, buf_1, 2, 1024) assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes # open the file to read ret = gkfs_clientLibc.open(file, os.O_RDONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # read the file ret = gkfs_clientLibc.preadv(file, len(buf_0), len(buf_1), 1024) assert ret.buf_0 == buf_0 assert ret.buf_1 == buf_1 assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes No newline at end of file tests/integration/operations/test_write_operations.py +47 −0 Original line number Diff line number Diff line Loading @@ -129,6 +129,53 @@ def test_write_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): assert ret.retval == 0 assert ret.statbuf.st_size == (len(str1) + len(str2) + len(str3)) def test_write(gkfs_daemon, gkfs_clientLibc): file = gkfs_daemon.mountdir / "file" ret = gkfs_clientLibc.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 buf = b'42' ret = gkfs_clientLibc.write(file, buf, len(buf)) assert ret.retval == len(buf) # Return the number of written bytes file_append = gkfs_daemon.mountdir / "file_append" ret = gkfs_clientLibc.open(file_append, os.O_CREAT | os.O_WRONLY | os.O_APPEND, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 str1 = b'Hello' str2 = b', World!' str3 = b' This is a test.\n' ret = gkfs_clientLibc.write(file_append, str1, len(str1), True) assert ret.retval == len(str1) ret = gkfs_clientLibc.stat(file_append) assert ret.retval == 0 assert ret.statbuf.st_size == len(str1) ret = gkfs_clientLibc.write(file_append, str2, len(str2), True) assert ret.retval == len(str2) ret = gkfs_clientLibc.stat(file_append) assert ret.retval == 0 assert ret.statbuf.st_size == (len(str1) + len(str2)) ret = gkfs_clientLibc.write(file_append, str3, len(str3), True) assert ret.retval == len(str3) ret = gkfs_clientLibc.stat(file_append) assert ret.retval == 0 assert ret.statbuf.st_size == (len(str1) + len(str2) + len(str3)) def test_pwrite(gkfs_daemon, gkfs_client): file = gkfs_daemon.mountdir / "file" Loading Loading
tests/integration/conftest.py +20 −1 Original line number Diff line number Diff line Loading @@ -34,7 +34,7 @@ from pathlib import Path from harness.logger import logger, initialize_logging, finalize_logging from harness.cli import add_cli_options, set_default_log_formatter from harness.workspace import Workspace, FileCreator from harness.gkfs import Daemon, Client, Proxy, ShellClient, FwdDaemon, FwdClient, ShellFwdClient, FwdDaemonCreator, FwdClientCreator from harness.gkfs import Daemon, Client, ClientLibc, Proxy, ShellClient, ShellClientLibc, FwdDaemon, FwdClient, ShellFwdClient, FwdDaemonCreator, FwdClientCreator from harness.reporter import report_test_status, report_test_headline, report_assertion_pass def pytest_configure(config): Loading Loading @@ -159,6 +159,16 @@ def gkfs_client_proxy(test_workspace): return Client(test_workspace, True) @pytest.fixture def gkfs_clientLibc(test_workspace): """ Sets up a gekkofs client environment so that operations (system calls, library calls, ...) can be requested from a co-running daemon. """ return ClientLibc(test_workspace) @pytest.fixture def gkfs_shell(test_workspace): """ Loading @@ -177,6 +187,15 @@ def gkfs_shell_proxy(test_workspace): return ShellClient(test_workspace,True) @pytest.fixture def gkfs_shellLibc(test_workspace): """ Sets up a gekkofs environment so that shell commands (stat, ls, mkdir, etc.) can be issued to a co-running daemon. """ return ShellClientLibc(test_workspace) @pytest.fixture def file_factory(test_workspace): """ Loading
tests/integration/conftest.template +20 −1 Original line number Diff line number Diff line Loading @@ -34,7 +34,7 @@ from pathlib import Path from harness.logger import logger, initialize_logging, finalize_logging from harness.cli import add_cli_options, set_default_log_formatter from harness.workspace import Workspace, FileCreator from harness.gkfs import Daemon, Client, Proxy, ShellClient, FwdDaemon, FwdClient, ShellFwdClient, FwdDaemonCreator, FwdClientCreator from harness.gkfs import Daemon, Client, ClientLibc, Proxy, ShellClient, ShellClientLibc, FwdDaemon, FwdClient, ShellFwdClient, FwdDaemonCreator, FwdClientCreator from harness.reporter import report_test_status, report_test_headline, report_assertion_pass def pytest_configure(config): Loading Loading @@ -159,6 +159,16 @@ def gkfs_client_proxy(test_workspace): return Client(test_workspace, True) @pytest.fixture def gkfs_clientLibc(test_workspace): """ Sets up a gekkofs client environment so that operations (system calls, library calls, ...) can be requested from a co-running daemon. """ return ClientLibc(test_workspace) @pytest.fixture def gkfs_shell(test_workspace): """ Loading @@ -177,6 +187,15 @@ def gkfs_shell_proxy(test_workspace): return ShellClient(test_workspace,True) @pytest.fixture def gkfs_shellLibc(test_workspace): """ Sets up a gekkofs environment so that shell commands (stat, ls, mkdir, etc.) can be issued to a co-running daemon. """ return ShellClientLibc(test_workspace) @pytest.fixture def file_factory(test_workspace): """ Loading
tests/integration/harness/gkfs.py +312 −3 Original line number Diff line number Diff line Loading @@ -40,6 +40,7 @@ from harness.cmd import CommandParser gkfs_daemon_cmd = 'gkfs_daemon' gkfs_client_cmd = 'gkfs.io' gkfs_client_lib_file = 'libgkfs_intercept.so' gkfs_client_lib_libc_file = 'libgkfs_libc_intercept.so' gkfs_hosts_file = 'gkfs_hosts.txt' gkfs_daemon_log_file = 'gkfs_daemon.log' gkfs_daemon_log_level = '100' Loading Loading @@ -255,10 +256,10 @@ class Daemon: def run(self): args = ['--mountdir', self.mountdir, '--rootdir', self.rootdir, args = ['--mountdir', self.mountdir.as_posix(), '--rootdir', self.rootdir.as_posix(), '-l', self._address, '--metadir', self._metadir, '--metadir', self._metadir.as_posix(), '--dbbackend', self._database, '--output-stats', self.logdir / 'stats.log', '--enable-collection', Loading Loading @@ -602,6 +603,87 @@ class Client: def cwd(self): return self._workspace.twd class ClientLibc: """ A class to represent a GekkoFS client process with a patched LD_PRELOAD. This class allows tests to interact with the file system using I/O-related function calls, be them system calls (e.g. read()) or glibc I/O functions (e.g. opendir()). """ def __init__(self, workspace): self._parser = IOParser() self._workspace = workspace self._cmd = sh.Command(gkfs_client_cmd, self._workspace.bindirs) self._env = os.environ.copy() libdirs = ':'.join( filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + [str(p) for p in self._workspace.libdirs])) # ensure the client interception library is available: # to avoid running code with potentially installed libraries, # it must be found in one (and only one) of the workspace's bindirs preloads = [] for d in self._workspace.bindirs: search_path = Path(d) / gkfs_client_lib_libc_file if search_path.exists(): preloads.append(search_path) if len(preloads) == 0: logger.error(f'No client libraries found in the test\'s binary directories:') pytest.exit("Aborted due to initialization error. Check test logs.") if len(preloads) != 1: logger.error(f'Multiple client libraries found in the test\'s binary directories:') for p in preloads: logger.error(f' {p}') logger.error(f'Make sure that only one copy of the client library is available.') pytest.exit("Aborted due to initialization error. Check test logs.") self._preload_library = preloads[0] self._patched_env = { 'LD_LIBRARY_PATH': libdirs, 'LD_PRELOAD': str(self._preload_library), 'LIBGKFS_HOSTS_FILE': str(self.cwd / gkfs_hosts_file), 'LIBGKFS_LOG': gkfs_client_log_level, 'LIBGKFS_LOG_OUTPUT': str(self._workspace.logdir / gkfs_client_log_file), 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @property def preload_library(self): """ Return the preload library detected for this client """ return self._preload_library def run(self, cmd, *args): logger.debug(f"running client") logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, list(args)))) logger.debug(f"patched env: {pformat(self._patched_env)}") out = self._cmd( [ cmd ] + list(args), _env = self._env, # _out=sys.stdout, # _err=sys.stderr, ) logger.debug(f"command output: {out.stdout}") return self._parser.parse(cmd, out.stdout) def __getattr__(self, name): return _proxy_exec(self, name) @property def cwd(self): return self._workspace.twd class ShellCommand: """ A wrapper class for sh.RunningCommand that allows seamlessly using all Loading Loading @@ -814,6 +896,233 @@ class ShellClient: found_cmd = shutil.which(cmd, path=':'.join(str(p) for p in self._search_paths) ) if not found_cmd: raise sh.CommandNotFound(cmd) self._cmd = sh.Command(found_cmd) logger.debug(f"running program") logger.debug(f"cmd: {cmd} {' '.join(str(a) for a in args)}") logger.debug(f"search_paths: {':'.join(str(p) for p in self._search_paths)}") logger.debug(f"timeout: {timeout} seconds") logger.debug(f"timeout_signal: {signal.Signals(timeout_signal).name}") logger.debug(f"patched env:\n{pformat(self._patched_env)}") # 'sh' raises an exception if the return code is not zero; # since we'd rather check for return codes explictly, we # whitelist all exit codes from 1 to 255 as 'ok' using the # _ok_code argument proc = self._cmd( args, _env = self._env, # _out=sys.stdout, # _err=sys.stderr, _timeout=timeout, _timeout_signal=timeout_signal, # _ok_code=list(range(0, 256)) ) logger.debug(f"program stdout: {proc.stdout}") logger.debug(f"program stderr: {proc.stderr}") return ShellCommand(cmd, proc) def __getattr__(self, name): return _proxy_exec(self, name) @property def cwd(self): return self._workspace.twd class ShellClientLibc: """ A class to represent a GekkoFS shell client process. This class allows tests to execute shell commands or scripts via bash -c on a GekkoFS instance. """ def __init__(self, workspace): self._workspace = workspace self._search_paths = _find_search_paths(self._workspace.bindirs) self._env = os.environ.copy() libdirs = ':'.join( filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + [str(p) for p in self._workspace.libdirs])) # ensure the client interception library is available: # to avoid running code with potentially installed libraries, # it must be found in one (and only one) of the workspace's bindirs preloads = [] for d in self._workspace.bindirs: search_path = Path(d) / gkfs_client_lib_libc_file if search_path.exists(): preloads.append(search_path) if len(preloads) != 1: logger.error(f'Multiple client libraries found in the test\'s binary directories:') for p in preloads: logger.error(f' {p}') logger.error(f'Make sure that only one copy of the client library is available.') pytest.exit("Aborted due to initialization error") self._preload_library = preloads[0] self._patched_env = { 'LD_LIBRARY_PATH' : libdirs, 'LD_PRELOAD' : str(self._preload_library), 'LIBGKFS_HOSTS_FILE' : str(self.cwd / gkfs_hosts_file), 'LIBGKFS_LOG' : gkfs_client_log_level, 'LIBGKFS_LOG_OUTPUT' : str(self._workspace.logdir / gkfs_client_log_file), 'LIBGKFS_LOG_SYSCALL_FILTER': gkfs_client_log_syscall_filter } self._env.update(self._patched_env) @property def patched_environ(self): """ Return the patched environment required to run a test as a string that can be prepended to a shell command. """ return ' '.join(f'{k}="{v}"' for k,v in self._patched_env.items()) def script(self, code, intercept_shell=True, timeout=60, timeout_signal=signal.SIGKILL): """ Execute a shell script passed as an argument in bash. For instance, the following snippet: mountdir = pathlib.Path('/tmp') file01 = 'file01' ShellClient().script( f''' expected_pathname={mountdir / file01} if [[ -e ${{expected_pathname}} ]]; then exit 0 fi exit 1 ''') transforms into: bash -c ' expected_pathname=/tmp/file01 if [[ -e ${expected_pathname} ]]; then exit 0 fi exit 1 ' Note that since we are using Python's f-strings, for variable expansions to work correctly, they need to be defined with double braces, e.g. ${{expected_pathname}}. Parameters ---------- code: `str` The script code to be passed to 'bash -c'. intercept_shell: `bool` Controls whether the shell executing the script should be executed with LD_PRELOAD=libgkfs_intercept.so (default: True). timeout: `int` How much time, in seconds, we should give the process to complete. If the process does not finish within the timeout, it will be sent the signal defined by `timeout_signal`. Default value: 60 timeout_signal: `int` The signal to be sent to the process if `timeout` is not None. Default value: signal.SIGKILL Returns ------- A sh.RunningCommand instance that allows interacting with the finished process. """ logger.debug(f"running bash") logger.debug(f"cmd: bash -c '{code}'") logger.debug(f"timeout: {timeout} seconds") logger.debug(f"timeout_signal: {signal.Signals(timeout_signal).name}") if intercept_shell: logger.debug(f"patched env: {self._patched_env}") self._cmd = sh.Command("bash") # 'sh' raises an exception if the return code is not zero; # since we'd rather check for return codes explictly, we # whitelist all exit codes from 1 to 255 as 'ok' using the # _ok_code argument return self._cmd('-c', code, _env = (self._env if intercept_shell else os.environ), # _out=sys.stdout, # _err=sys.stderr, _timeout=timeout, _timeout_signal=timeout_signal, # _ok_code=list(range(0, 256)) ) def run(self, cmd, *args, timeout=60, timeout_signal=signal.SIGKILL): """ Execute a shell command with arguments. For example, the following snippet: mountdir = pathlib.Path('/tmp') file01 = 'file01' ShellClient().stat('--terse', mountdir / file01) transforms into: bash -c 'stat --terse /tmp/file01' Parameters: ----------- cmd: `str` The command to execute. args: `list` The list of arguments for the command. timeout: `number` How much time, in seconds, we should give the process to complete. If the process does not finish within the timeout, it will be sent the signal defined by `timeout_signal`. Default value: 60 timeout_signal: `int` The signal to be sent to the process if `timeout` is not None. Default value: signal.SIGKILL Returns ------- A ShellCommand instance that allows interacting with the finished process. Note that ShellCommand wraps sh.RunningCommand and adds s extra properties to it. """ found_cmd = shutil.which(cmd, path=':'.join(str(p) for p in self._search_paths) ) Loading
tests/integration/operations/test_read_operations.py +128 −0 Original line number Diff line number Diff line Loading @@ -192,3 +192,131 @@ def test_preadv(gkfs_daemon, gkfs_client): assert ret.buf_0 == buf_0 assert ret.buf_1 == buf_1 assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes def test_read_libc(gkfs_daemon, gkfs_clientLibc): file = gkfs_daemon.mountdir / "file" # create a file in gekkofs ret = gkfs_clientLibc.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # write a buffer we know buf = b'42' ret = gkfs_clientLibc.write(file, buf, len(buf)) assert ret.retval == len(buf) # Return the number of written bytes # open the file to read ret = gkfs_clientLibc.open(file, os.O_RDONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # read the file ret = gkfs_clientLibc.read(file, len(buf)) assert ret.buf == buf assert ret.retval == len(buf) # Return the number of read bytes def test_pread_libc(gkfs_daemon, gkfs_clientLibc): file = gkfs_daemon.mountdir / "file" # create a file in gekkofs ret = gkfs_clientLibc.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # write a buffer we know buf = b'42' ret = gkfs_clientLibc.pwrite(file, buf, len(buf), 1024) assert ret.retval == len(buf) # Return the number of written bytes # open the file to read ret = gkfs_clientLibc.open(file, os.O_RDONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # read the file at offset 1024 ret = gkfs_clientLibc.pread(file, len(buf), 1024) assert ret.buf == buf assert ret.retval == len(buf) # Return the number of read bytes @pytest.mark.skip(reason="readv not implemented in libc") def test_readv_libc(gkfs_daemon, gkfs_clientLibc): file = gkfs_daemon.mountdir / "file" # create a file in gekkofs ret = gkfs_clientLibc.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # write a buffer we know buf_0 = b'42' buf_1 = b'24' ret = gkfs_clientLibc.writev(file, buf_0, buf_1, 2) assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes # open the file to read ret = gkfs_clientLibc.open(file, os.O_RDONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # read the file ret = gkfs_clientLibc.readv(file, len(buf_0), len(buf_1)) assert ret.buf_0 == buf_0 assert ret.buf_1 == buf_1 assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes @pytest.mark.skip(reason="preadv not implemented in libc") def test_preadv_libc(gkfs_daemon, gkfs_clientLibc): file = gkfs_daemon.mountdir / "file" # create a file in gekkofs ret = gkfs_clientLibc.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # write a buffer we know buf_0 = b'42' buf_1 = b'24' ret = gkfs_clientLibc.pwritev(file, buf_0, buf_1, 2, 1024) assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes # open the file to read ret = gkfs_clientLibc.open(file, os.O_RDONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # read the file ret = gkfs_clientLibc.preadv(file, len(buf_0), len(buf_1), 1024) assert ret.buf_0 == buf_0 assert ret.buf_1 == buf_1 assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes No newline at end of file
tests/integration/operations/test_write_operations.py +47 −0 Original line number Diff line number Diff line Loading @@ -129,6 +129,53 @@ def test_write_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): assert ret.retval == 0 assert ret.statbuf.st_size == (len(str1) + len(str2) + len(str3)) def test_write(gkfs_daemon, gkfs_clientLibc): file = gkfs_daemon.mountdir / "file" ret = gkfs_clientLibc.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 buf = b'42' ret = gkfs_clientLibc.write(file, buf, len(buf)) assert ret.retval == len(buf) # Return the number of written bytes file_append = gkfs_daemon.mountdir / "file_append" ret = gkfs_clientLibc.open(file_append, os.O_CREAT | os.O_WRONLY | os.O_APPEND, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 str1 = b'Hello' str2 = b', World!' str3 = b' This is a test.\n' ret = gkfs_clientLibc.write(file_append, str1, len(str1), True) assert ret.retval == len(str1) ret = gkfs_clientLibc.stat(file_append) assert ret.retval == 0 assert ret.statbuf.st_size == len(str1) ret = gkfs_clientLibc.write(file_append, str2, len(str2), True) assert ret.retval == len(str2) ret = gkfs_clientLibc.stat(file_append) assert ret.retval == 0 assert ret.statbuf.st_size == (len(str1) + len(str2)) ret = gkfs_clientLibc.write(file_append, str3, len(str3), True) assert ret.retval == len(str3) ret = gkfs_clientLibc.stat(file_append) assert ret.retval == 0 assert ret.statbuf.st_size == (len(str1) + len(str2) + len(str3)) def test_pwrite(gkfs_daemon, gkfs_client): file = gkfs_daemon.mountdir / "file" Loading
