Resolve "syscall deadlocks MPI Dynamic Process Management" (221c8596) · Commits · hpc / gekkofs

src/client/intercept.cpp

+136 −0

Original line number	Diff line number	Diff line
		@@ -46,6 +46,7 @@
		#include <fmt/format.h>

		#include <cerrno>
		#include <cstring>

		extern "C" {
		#include <syscall.h>
		@@ -102,6 +103,136 @@ get_current_syscall_info() {
		return saved_syscall_info;
		}

		bool
		starts_with(const char* path, const char* prefix) {
		return path != nullptr &&
		std::strncmp(path, prefix, std::strlen(prefix)) == 0;
		}

		bool
		is_mpi_runtime_path(const char* path) {
		// Keep MPI/PMIx/PRRTE runtime-private files out of the GekkoFS hook path.
		// They live outside the GekkoFS mount and are hot during dynamic process
		// management; if more launchers need this, replace the prefixes with a
		// configurable bypass list.
		return starts_with(path, "/tmp/ompi.") \|\|
		starts_with(path, "/tmp/openmpi-sessions-") \|\|
		starts_with(path, "/tmp/pmix.") \|\| starts_with(path, "/tmp/prte.") \|\|
		starts_with(path, "/dev/shm/ompi") \|\|
		starts_with(path, "/dev/shm/openmpi") \|\|
		starts_with(path, "/dev/shm/pmix") \|\|
		starts_with(path, "/dev/shm/prte") \|\|
		starts_with(path, "/run/user/");
		}

		const char*
		first_path_arg(long syscall_number, long arg0, long arg1, long arg3) {
		switch(syscall_number) {
		#ifdef SYS_open
		case SYS_open:
		#endif
		#ifdef SYS_creat
		case SYS_creat:
		#endif
		#ifdef SYS_stat
		case SYS_stat:
		#endif
		#ifdef SYS_lstat
		case SYS_lstat:
		#endif
		#ifdef SYS_access
		case SYS_access:
		#endif
		#ifdef SYS_unlink
		case SYS_unlink:
		#endif
		#ifdef SYS_rmdir
		case SYS_rmdir:
		#endif
		#ifdef SYS_chmod
		case SYS_chmod:
		#endif
		#ifdef SYS_chown
		case SYS_chown:
		#endif
		#ifdef SYS_lchown
		case SYS_lchown:
		#endif
		#ifdef SYS_readlink
		case SYS_readlink:
		#endif
		#ifdef SYS_truncate
		case SYS_truncate:
		#endif
		#ifdef SYS_statfs
		case SYS_statfs:
		#endif
		return reinterpret_cast<const char*>(arg0);

		case SYS_openat:
		#ifdef SYS_openat2
		case SYS_openat2:
		#endif
		case SYS_newfstatat:
		#ifdef STATX_TYPE
		case SYS_statx:
		#endif
		case SYS_unlinkat:
		case SYS_symlinkat:
		case SYS_faccessat:
		#ifdef SYS_faccessat2
		case SYS_faccessat2:
		#endif
		case SYS_mkdirat:
		case SYS_fchownat:
		case SYS_fchmodat:
		case SYS_fchmodat2:
		case SYS_readlinkat:
		#ifdef SYS_renameat
		case SYS_renameat:
		#endif
		#ifdef SYS_renameat2
		case SYS_renameat2:
		#endif
		return reinterpret_cast<const char*>(arg1);

		#ifdef SYS_rename
		case SYS_rename:
		return reinterpret_cast<const char*>(arg0);
		#endif

		default:
		return nullptr;
		}
		}

		const char*
		second_path_arg(long syscall_number, long arg1, long arg3) {
		switch(syscall_number) {
		#ifdef SYS_rename
		case SYS_rename:
		return reinterpret_cast<const char*>(arg1);
		#endif
		#ifdef SYS_renameat
		case SYS_renameat:
		#endif
		#ifdef SYS_renameat2
		case SYS_renameat2:
		#endif
		return reinterpret_cast<const char*>(arg3);
		default:
		return nullptr;
		}
		}

		bool
		should_bypass_mpi_runtime_path(long syscall_number, long arg0, long arg1,
		long arg3) {
		return is_mpi_runtime_path(
		first_path_arg(syscall_number, arg0, arg1, arg3)) \|\|
		is_mpi_runtime_path(second_path_arg(syscall_number, arg1, arg3));
		}

		#ifdef SYS_close_range
		std::vector<int>
		get_open_fds() {
		@@ -531,6 +662,11 @@ inline int
		hook(long syscall_number, long arg0, long arg1, long arg2, long arg3, long arg4,
		long arg5, long* result) {

		if(should_bypass_mpi_runtime_path(syscall_number, arg0, arg1, arg3)) {
		::reset_current_syscall_info();
		return gkfs::syscall::forward_to_kernel;
		}

		#if defined(GKFS_ENABLE_LOGGING) && defined(GKFS_DEBUG_BUILD)
		const long args[gkfs::syscall::MAX_ARGS] = {arg0, arg1, arg2,
		arg3, arg4, arg5};

tests/integration/syscalls/test_interceptor_runtime_bypass.py

0 → 100644

+36 −0

Original line number	Diff line number	Diff line
		import os
		from pathlib import Path


		def test_preload_bypasses_mpi_runtime_paths(gkfs_daemon, gkfs_shell):
		"""MPI/PMIx runtime files must not enter GekkoFS path hooks.

		Bug 8 showed that the preload interceptor sitting in front of MPI/PMIx
		runtime syscalls can deadlock dynamic process management. This local test
		cannot reproduce Slurm/PMIx, but it does lock in the required boundary:
		runtime-private paths outside the GekkoFS mount are forwarded before
		hook_openat()/relativize touches GekkoFS state.
		"""

		runtime_dir = Path("/tmp") / f"ompi.{os.getpid()}.gkfs-bug8"
		runtime_file = runtime_dir / "pmix-rendezvous"
		client_log = Path(gkfs_shell._patched_env["LIBGKFS_LOG_OUTPUT"])

		client_log.unlink(missing_ok=True)

		cmd = gkfs_shell.script(
		f"""
		set -eu
		rm -rf '{runtime_dir}'
		mkdir -p '{runtime_dir}'
		: > '{runtime_file}'
		stat '{runtime_file}' >/dev/null
		rm -rf '{runtime_dir}'
		""",
		timeout=10,
		)

		assert cmd.exit_code == 0, cmd.stderr.decode(errors="replace")

		log = client_log.read_text(errors="replace") if client_log.exists() else ""
		assert str(runtime_file) not in log