From 9cce211df96c378b0b1d4cb05123aff4192a2271 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 10 Dec 2020 16:07:56 +0100 Subject: [PATCH 1/4] ShellClient now considers _workspace.bindirs when searching for programs --- tests/integration/harness/gkfs.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index 07373ddb0..bcec22a2a 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -140,6 +140,32 @@ def _process_exists(pid): return True +def _find_search_paths(additional_paths=None): + """ + Return the entire list of search paths available to the process. If + additional_paths is not provided, $PATH env is returned. + + Parameters + ---------- + additional_paths: `list` + If provided, additional paths that should be used for searching before + falling back to the contents of $PATH. + + Returns + ------- + A list containing the paths that should be searched for commands. + """ + + paths_to_search = [] + + if isinstance(additional_paths, (tuple, list)): + paths_to_search.extend(additional_paths) + + env_paths = os.environ.get("PATH", "").split(os.pathsep) + paths_to_search.extend(env_paths) + + return paths_to_search + class FwdDaemonCreator: """ Factory that allows tests to create forwarding daemons in a workspace. @@ -439,6 +465,7 @@ class ShellClient: def __init__(self, workspace): self._workspace = workspace + self._search_paths = _find_search_paths(self._workspace.bindirs) self._cmd = sh.Command("bash") self._env = os.environ.copy() @@ -608,9 +635,12 @@ class ShellClient: extra properties to it. """ + cmd = sh.which(cmd, self._search_paths) + bash_c_args = f"{cmd} {' '.join(str(a) for a in args)}" logger.debug(f"running bash") logger.debug(f"cmd: bash -c '{bash_c_args}'") + logger.debug(f"search_paths: {':'.join(str(p) for p in self._search_paths)}") logger.debug(f"timeout: {timeout} seconds") logger.debug(f"timeout_signal: {signal.Signals(timeout_signal).name}") logger.debug(f"patched env:\n{pformat(self._patched_env)}") -- GitLab From 085d06654aa064b0510781aeee3b3fa3eb2e30f5 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 10 Dec 2020 16:08:20 +0100 Subject: [PATCH 2/4] Add examples/gfind as a binary directory for tests --- tests/integration/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 75d3c79cd..1321a2a2f 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -5,6 +5,7 @@ gkfs_enable_python_testing( BINARY_DIRECTORIES ${CMAKE_BINARY_DIR}/src/daemon/ ${CMAKE_BINARY_DIR}/src/client/ ${CMAKE_BINARY_DIR}/tests/integration/harness/ + ${CMAKE_BINARY_DIR}/examples/gfind/ LIBRARY_PREFIX_DIRECTORIES ${CMAKE_PREFIX_PATH} ) -- GitLab From 6569690f2ef77dcd3fe33b555598fd240a7d79fa Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 10 Dec 2020 22:22:18 +0100 Subject: [PATCH 3/4] Replace os.popen(sfind, ...) with gkfs_shell.sfind(...) in test_extended --- tests/integration/directories/test_directories.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/integration/directories/test_directories.py b/tests/integration/directories/test_directories.py index c83789f13..592fbee8c 100644 --- a/tests/integration/directories/test_directories.py +++ b/tests/integration/directories/test_directories.py @@ -220,11 +220,16 @@ def test_extended(gkfs_daemon, gkfs_shell, gkfs_client): ret = gkfs_client.write(file_a, buf, 1) assert ret.retval == 1 - preload = 'LIBGKFS_HOSTS_FILE='+str(gkfs_client._patched_env['LIBGKFS_HOSTS_FILE'])+' LD_PRELOAD='+str(gkfs_client._preload_library) - stream = os.popen(preload+' '+str(gkfs_daemon._workspace.bindirs[1])+'/sfind '+str(topdir)+' -M '+str(gkfs_daemon.mountdir)+' -S 1 -name "*_k*"'); - output = stream.read() - assert output == "MATCHED 0/4\n" - + + cmd = gkfs_shell.sfind( + topdir, + f'-M {gkfs_daemon.mountdir}', + '-S 1', + '-name "*_k*"' + ) + + assert cmd.exit_code == 0 + assert cmd.stdout.decode() == "MATCHED 0/4\n" @pytest.mark.skip(reason="invalid errno returned on success") @pytest.mark.parametrize("directory_path", -- GitLab From 0b8d824cf3f084cca941ba15449a657c08bdae3e Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 11 Dec 2020 20:50:13 +0100 Subject: [PATCH 4/4] Avoid relying on 'bash -c' for simple programs --- .../directories/test_directories.py | 9 +++++--- tests/integration/harness/gkfs.py | 23 +++++++++++++------ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/integration/directories/test_directories.py b/tests/integration/directories/test_directories.py index 592fbee8c..4a60617dd 100644 --- a/tests/integration/directories/test_directories.py +++ b/tests/integration/directories/test_directories.py @@ -223,9 +223,12 @@ def test_extended(gkfs_daemon, gkfs_shell, gkfs_client): cmd = gkfs_shell.sfind( topdir, - f'-M {gkfs_daemon.mountdir}', - '-S 1', - '-name "*_k*"' + '-M', + gkfs_daemon.mountdir, + '-S', + 1, + '-name', + '*_k*' ) assert cmd.exit_code == 0 diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index bcec22a2a..056bb1463 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -466,7 +466,6 @@ class ShellClient: def __init__(self, workspace): self._workspace = workspace self._search_paths = _find_search_paths(self._workspace.bindirs) - self._cmd = sh.Command("bash") self._env = os.environ.copy() libdirs = ':'.join( @@ -579,6 +578,8 @@ class ShellClient: if intercept_shell: logger.debug(f"patched env: {self._patched_env}") + self._cmd = sh.Command("bash") + # 'sh' raises an exception if the return code is not zero; # since we'd rather check for return codes explictly, we # whitelist all exit codes from 1 to 255 as 'ok' using the @@ -635,22 +636,27 @@ class ShellClient: extra properties to it. """ - cmd = sh.which(cmd, self._search_paths) + found_cmd = sh.which(cmd, self._search_paths) - bash_c_args = f"{cmd} {' '.join(str(a) for a in args)}" - logger.debug(f"running bash") - logger.debug(f"cmd: bash -c '{bash_c_args}'") + if not found_cmd: + raise sh.CommandNotFound(cmd) + + self._cmd = sh.Command(found_cmd) + + logger.debug(f"running program") + logger.debug(f"cmd: {cmd} {' '.join(str(a) for a in args)}") logger.debug(f"search_paths: {':'.join(str(p) for p in self._search_paths)}") logger.debug(f"timeout: {timeout} seconds") logger.debug(f"timeout_signal: {signal.Signals(timeout_signal).name}") logger.debug(f"patched env:\n{pformat(self._patched_env)}") + # 'sh' raises an exception if the return code is not zero; # since we'd rather check for return codes explictly, we # whitelist all exit codes from 1 to 255 as 'ok' using the # _ok_code argument - proc = self._cmd('-c', - bash_c_args, + proc = self._cmd( + args, _env = self._env, # _out=sys.stdout, # _err=sys.stderr, @@ -659,6 +665,9 @@ class ShellClient: # _ok_code=list(range(0, 256)) ) + logger.debug(f"program stdout: {proc.stdout}") + logger.debug(f"program stderr: {proc.stderr}") + return ShellCommand(cmd, proc) def __getattr__(self, name): -- GitLab