diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index e6d470d7480809693f50c6111e33fe71393d3658..d9b9c9c326bdde47085633e04561cb18a593d22c 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -46,6 +46,7 @@ #include #include +#include extern "C" { #include @@ -102,6 +103,136 @@ get_current_syscall_info() { return saved_syscall_info; } +bool +starts_with(const char* path, const char* prefix) { + return path != nullptr && + std::strncmp(path, prefix, std::strlen(prefix)) == 0; +} + +bool +is_mpi_runtime_path(const char* path) { + // Keep MPI/PMIx/PRRTE runtime-private files out of the GekkoFS hook path. + // They live outside the GekkoFS mount and are hot during dynamic process + // management; if more launchers need this, replace the prefixes with a + // configurable bypass list. + return starts_with(path, "/tmp/ompi.") || + starts_with(path, "/tmp/openmpi-sessions-") || + starts_with(path, "/tmp/pmix.") || starts_with(path, "/tmp/prte.") || + starts_with(path, "/dev/shm/ompi") || + starts_with(path, "/dev/shm/openmpi") || + starts_with(path, "/dev/shm/pmix") || + starts_with(path, "/dev/shm/prte") || + starts_with(path, "/run/user/"); +} + +const char* +first_path_arg(long syscall_number, long arg0, long arg1, long arg3) { + switch(syscall_number) { +#ifdef SYS_open + case SYS_open: +#endif +#ifdef SYS_creat + case SYS_creat: +#endif +#ifdef SYS_stat + case SYS_stat: +#endif +#ifdef SYS_lstat + case SYS_lstat: +#endif +#ifdef SYS_access + case SYS_access: +#endif +#ifdef SYS_unlink + case SYS_unlink: +#endif +#ifdef SYS_rmdir + case SYS_rmdir: +#endif +#ifdef SYS_chmod + case SYS_chmod: +#endif +#ifdef SYS_chown + case SYS_chown: +#endif +#ifdef SYS_lchown + case SYS_lchown: +#endif +#ifdef SYS_readlink + case SYS_readlink: +#endif +#ifdef SYS_truncate + case SYS_truncate: +#endif +#ifdef SYS_statfs + case SYS_statfs: +#endif + return reinterpret_cast(arg0); + + case SYS_openat: +#ifdef SYS_openat2 + case SYS_openat2: +#endif + case SYS_newfstatat: +#ifdef STATX_TYPE + case SYS_statx: +#endif + case SYS_unlinkat: + case SYS_symlinkat: + case SYS_faccessat: +#ifdef SYS_faccessat2 + case SYS_faccessat2: +#endif + case SYS_mkdirat: + case SYS_fchownat: + case SYS_fchmodat: + case SYS_fchmodat2: + case SYS_readlinkat: +#ifdef SYS_renameat + case SYS_renameat: +#endif +#ifdef SYS_renameat2 + case SYS_renameat2: +#endif + return reinterpret_cast(arg1); + +#ifdef SYS_rename + case SYS_rename: + return reinterpret_cast(arg0); +#endif + + default: + return nullptr; + } +} + +const char* +second_path_arg(long syscall_number, long arg1, long arg3) { + switch(syscall_number) { +#ifdef SYS_rename + case SYS_rename: + return reinterpret_cast(arg1); +#endif +#ifdef SYS_renameat + case SYS_renameat: +#endif +#ifdef SYS_renameat2 + case SYS_renameat2: +#endif + return reinterpret_cast(arg3); + default: + return nullptr; + } +} + +bool +should_bypass_mpi_runtime_path(long syscall_number, long arg0, long arg1, + long arg3) { + return is_mpi_runtime_path( + first_path_arg(syscall_number, arg0, arg1, arg3)) || + is_mpi_runtime_path(second_path_arg(syscall_number, arg1, arg3)); +} + #ifdef SYS_close_range std::vector get_open_fds() { @@ -531,6 +662,11 @@ inline int hook(long syscall_number, long arg0, long arg1, long arg2, long arg3, long arg4, long arg5, long* result) { + if(should_bypass_mpi_runtime_path(syscall_number, arg0, arg1, arg3)) { + ::reset_current_syscall_info(); + return gkfs::syscall::forward_to_kernel; + } + #if defined(GKFS_ENABLE_LOGGING) && defined(GKFS_DEBUG_BUILD) const long args[gkfs::syscall::MAX_ARGS] = {arg0, arg1, arg2, arg3, arg4, arg5}; diff --git a/tests/integration/syscalls/test_interceptor_runtime_bypass.py b/tests/integration/syscalls/test_interceptor_runtime_bypass.py new file mode 100644 index 0000000000000000000000000000000000000000..b00d8a19fd7728141202d4b1cefc5489295ba265 --- /dev/null +++ b/tests/integration/syscalls/test_interceptor_runtime_bypass.py @@ -0,0 +1,36 @@ +import os +from pathlib import Path + + +def test_preload_bypasses_mpi_runtime_paths(gkfs_daemon, gkfs_shell): + """MPI/PMIx runtime files must not enter GekkoFS path hooks. + + Bug 8 showed that the preload interceptor sitting in front of MPI/PMIx + runtime syscalls can deadlock dynamic process management. This local test + cannot reproduce Slurm/PMIx, but it does lock in the required boundary: + runtime-private paths outside the GekkoFS mount are forwarded before + hook_openat()/relativize touches GekkoFS state. + """ + + runtime_dir = Path("/tmp") / f"ompi.{os.getpid()}.gkfs-bug8" + runtime_file = runtime_dir / "pmix-rendezvous" + client_log = Path(gkfs_shell._patched_env["LIBGKFS_LOG_OUTPUT"]) + + client_log.unlink(missing_ok=True) + + cmd = gkfs_shell.script( + f""" + set -eu + rm -rf '{runtime_dir}' + mkdir -p '{runtime_dir}' + : > '{runtime_file}' + stat '{runtime_file}' >/dev/null + rm -rf '{runtime_dir}' + """, + timeout=10, + ) + + assert cmd.exit_code == 0, cmd.stderr.decode(errors="replace") + + log = client_log.read_text(errors="replace") if client_log.exists() else "" + assert str(runtime_file) not in log