perf(client): reduce interception overhead for external paths and descriptors (afbd67d2) · Commits · hpc / gekkofs

include/client/preload_context.hpp

+14 −0

Changes for include/client/preload_context.hpp: 14 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -207,6 +207,7 @@ private:
		mutable std::unordered_map<std::string, StatCacheEntry> stat_cache_;

		std::string cwd_;
		bool cwd_is_internal_{false};
		std::vector<std::string> mountdir_components_;
		std::string mountdir_;

		@@ -326,6 +327,19 @@ public:
		const std::string&
		cwd() const;

		bool
		cwd_is_internal() const noexcept;

		const char*
		mountdir_cstr() const noexcept {
		return mountdir_.c_str();
		}

		size_t
		mountdir_size() const noexcept {
		return mountdir_.size();
		}

		std::shared_ptr<const HostEndpointTable>
		hosts() const;

src/client/intercept.cpp

+225 −55

Changes for src/client/intercept.cpp: 225 added lines, 55 removed lines.
Original line number	Diff line number	Diff line
		@@ -83,13 +83,7 @@ namespace {

		std::atomic<bool> auxiliary_hooks_enabled{true};

		static pthread_key_t reentrance_guard_key;
		static pthread_once_t key_once_control = PTHREAD_ONCE_INIT;
		static void
		make_key() {
		// This function will be called exactly once per process.
		pthread_key_create(&reentrance_guard_key, NULL);
		}
		static thread_local bool in_gkfs_hook = false;

		thread_local gkfs::syscall::info saved_syscall_info;

		@@ -230,12 +224,214 @@ second_path_arg(long syscall_number, long arg1, long arg3) {
		}
		}

		static inline bool
		is_system_path(const char* path) {
		if(path == nullptr \|\| path[0] != '/') {
		return false;
		}
		if(!CTX) {
		return false;
		}
		const auto& mdir = CTX->mountdir();
		switch(path[1]) {
		case 'u': // /usr
		if(mdir.rfind("/usr", 0) == 0)
		return false;
		return (path[2] == 's' && path[3] == 'r' &&
		(path[4] == '/' \|\| path[4] == '\0'));
		case 'l': // /lib, /lib64
		if(mdir.rfind("/lib", 0) == 0)
		return false;
		if(path[2] == 'i' && path[3] == 'b') {
		return path[4] == '/' \|\| path[4] == '\0' \|\|
		(path[4] == '6' && path[5] == '4' &&
		(path[6] == '/' \|\| path[6] == '\0'));
		}
		return false;
		case 'e': // /etc
		if(mdir.rfind("/etc", 0) == 0)
		return false;
		return (path[2] == 't' && path[3] == 'c' &&
		(path[4] == '/' \|\| path[4] == '\0'));
		case 'p': // /proc
		if(mdir.rfind("/proc", 0) == 0)
		return false;
		return (path[2] == 'r' && path[3] == 'o' && path[4] == 'c' &&
		(path[5] == '/' \|\| path[5] == '\0'));
		case 's': // /sys, /sbin
		if(mdir.rfind("/sys", 0) == 0 \|\| mdir.rfind("/sbin", 0) == 0)
		return false;
		if(path[2] == 'y' && path[3] == 's' &&
		(path[4] == '/' \|\| path[4] == '\0')) {
		return true;
		}
		return (path[2] == 'b' && path[3] == 'i' && path[4] == 'n' &&
		(path[5] == '/' \|\| path[5] == '\0'));
		case 'd': // /dev
		if(mdir.rfind("/dev", 0) == 0)
		return false;
		return (path[2] == 'e' && path[3] == 'v' &&
		(path[4] == '/' \|\| path[4] == '\0'));
		case 'b': // /bin
		if(mdir.rfind("/bin", 0) == 0)
		return false;
		return (path[2] == 'i' && path[3] == 'n' &&
		(path[4] == '/' \|\| path[4] == '\0'));
		case 'r': // /run
		if(mdir.rfind("/run", 0) == 0)
		return false;
		return (path[2] == 'u' && path[3] == 'n' &&
		(path[4] == '/' \|\| path[4] == '\0'));
		case 'v': // /var
		if(mdir.rfind("/var", 0) == 0)
		return false;
		return (path[2] == 'a' && path[3] == 'r' &&
		(path[4] == '/' \|\| path[4] == '\0'));
		default:
		return false;
		}
		}

		static inline bool
		is_definitely_external_path(int dirfd, const char* path) {
		if(path == nullptr) {
		return false;
		}
		if(is_system_path(path) \|\| is_mpi_runtime_path(path)) {
		return true;
		}
		if(!CTX) {
		return false;
		}
		// Absolute path
		if(path[0] == '/') {
		const auto& mdir = CTX->mountdir();
		const size_t mlen = mdir.size();
		if(mlen == 0) {
		return true;
		}
		if(std::strncmp(path, mdir.c_str(), mlen) == 0) {
		char next = path[mlen];
		if(next == '\0' \|\| next == '/') {
		return false; // Candidate for GekkoFS
		}
		}
		// Does not match mountdir prefix
		if(std::strstr(path, "..") == nullptr) {
		return true;
		}
		return false;
		}
		// Relative path to an external directory descriptor
		if(dirfd != AT_FDCWD && dirfd >= 0) {
		if(CTX->file_map() && !CTX->file_map()->exist(dirfd)) {
		if(std::strstr(path, "..") == nullptr) {
		return true;
		}
		}
		return false;
		}
		// Relative path to AT_FDCWD when CWD is not in GekkoFS
		if(!CTX->cwd_is_internal()) {
		if(std::strstr(path, "..") == nullptr) {
		return true;
		}
		}
		return false;
		}

		static inline int
		first_dirfd_arg(long syscall_number, long arg0) {
		switch(syscall_number) {
		case SYS_openat:
		#ifdef SYS_openat2
		case SYS_openat2:
		#endif
		case SYS_newfstatat:
		#ifdef STATX_TYPE
		case SYS_statx:
		#endif
		case SYS_unlinkat:
		case SYS_faccessat:
		#ifdef SYS_faccessat2
		case SYS_faccessat2:
		#endif
		case SYS_mkdirat:
		case SYS_fchownat:
		case SYS_fchmodat:
		case SYS_fchmodat2:
		case SYS_readlinkat:
		#ifdef SYS_renameat
		case SYS_renameat:
		#endif
		#ifdef SYS_renameat2
		case SYS_renameat2:
		#endif
		return static_cast<int>(arg0);

		default:
		return AT_FDCWD;
		}
		}

		bool
		should_bypass_mpi_runtime_path(long syscall_number, long arg0, long arg1,
		should_bypass_syscall(long syscall_number, long arg0, long arg1, long arg2,
		long arg3) {
		return is_mpi_runtime_path(
		first_path_arg(syscall_number, arg0, arg1, arg3)) \|\|
		is_mpi_runtime_path(second_path_arg(syscall_number, arg1, arg3));
		if(!CTX) {
		return true;
		}

		const char* path1 = first_path_arg(syscall_number, arg0, arg1, arg3);
		const char* path2 = second_path_arg(syscall_number, arg1, arg3);

		// Fast path for pathname syscalls
		if(path1 != nullptr) {
		int dirfd1 = first_dirfd_arg(syscall_number, arg0);
		if(!is_definitely_external_path(dirfd1, path1)) {
		return false;
		}
		if(path2 != nullptr) {
		int dirfd2 = (syscall_number == SYS_renameat
		#ifdef SYS_renameat2
		\|\| syscall_number == SYS_renameat2
		#endif
		)
		? static_cast<int>(arg2)
		: AT_FDCWD;
		return is_definitely_external_path(dirfd2, path2);
		}
		return true;
		}

		// Fast bypass for fd-only syscalls when GekkoFS has 0 open files
		switch(syscall_number) {
		case SYS_read:
		case SYS_write:
		case SYS_pread64:
		case SYS_pwrite64:
		case SYS_readv:
		case SYS_writev:
		case SYS_preadv:
		case SYS_pwritev:
		case SYS_lseek:
		case SYS_fstat:
		case SYS_ftruncate:
		case SYS_fchmod:
		case SYS_fchown:
		case SYS_fsync:
		case SYS_fdatasync:
		case SYS_flock:
		case SYS_getdents:
		case SYS_getdents64:
		if(CTX->file_map() && CTX->file_map()->empty()) {
		return true;
		}
		break;
		default:
		break;
		}

		return false;
		}

		#ifdef SYS_close_range
		@@ -667,7 +863,7 @@ inline int
		hook(long syscall_number, long arg0, long arg1, long arg2, long arg3, long arg4,
		long arg5, long* result) {

		if(should_bypass_mpi_runtime_path(syscall_number, arg0, arg1, arg3)) {
		if(should_bypass_syscall(syscall_number, arg0, arg1, arg2, arg3)) {
		::reset_current_syscall_info();
		return gkfs::syscall::forward_to_kernel;
		}
		@@ -686,8 +882,7 @@ hook(long syscall_number, long arg0, long arg1, long arg2, long arg3, long arg4,

		case SYS_execve: {
		// If we do not set this to false, we are in trouble with vforks
		pthread_once(&key_once_control, make_key);
		pthread_setspecific(reentrance_guard_key, NULL);
		in_gkfs_hook = false;
		const char* pathname = reinterpret_cast<const char*>(arg0);
		std::string resolved_exec;
		if(pathname != nullptr) {
		@@ -725,8 +920,7 @@ hook(long syscall_number, long arg0, long arg1, long arg2, long arg3, long arg4,
		#ifdef SYS_execveat
		case SYS_execveat: {
		// If we do not set this to false, we are in trouble with vforks
		pthread_once(&key_once_control, make_key);
		pthread_setspecific(reentrance_guard_key, NULL);
		in_gkfs_hook = false;
		int dirfd = static_cast<int>(arg0);
		const char* pathname = reinterpret_cast<const char*>(arg1);
		std::string resolved_exec;
		@@ -1347,17 +1541,13 @@ hook_forwarded_syscall(long syscall_number, long arg0, long arg1, long arg2,
		if(!auxiliary_hooks_enabled.load(std::memory_order_acquire))
		return;

		pthread_once(&key_once_control, make_key);
		if(pthread_getspecific(reentrance_guard_key) != NULL) {
		if(in_gkfs_hook) {
		return;
		}
		pthread_setspecific(
		reentrance_guard_key,
		reinterpret_cast<void*>(1)); // cppcheck-suppress intToPointerCast

		in_gkfs_hook = true;

		if(::get_current_syscall_info() == gkfs::syscall::no_info) {
		pthread_setspecific(reentrance_guard_key, NULL);
		in_gkfs_hook = false;
		return;
		}

		@@ -1371,7 +1561,7 @@ hook_forwarded_syscall(long syscall_number, long arg0, long arg1, long arg2,

		::reset_current_syscall_info();

		pthread_setspecific(reentrance_guard_key, NULL);
		in_gkfs_hook = false;
		}

		void
		@@ -1389,15 +1579,10 @@ hook_clone_at_child(unsigned long flags, void* child_stack, int* ptid,
		static_cast<long>(newtls), 0};
		#endif

		pthread_once(&key_once_control, make_key);
		pthread_setspecific(
		reentrance_guard_key,
		reinterpret_cast<void*>(1)); // cppcheck-suppress intToPointerCast

		in_gkfs_hook = true;
		LOG(SYSCALL, ::get_current_syscall_info() \| gkfs::syscall::executed,
		SYS_clone, args, 0);

		pthread_setspecific(reentrance_guard_key, NULL);
		in_gkfs_hook = false;
		}

		void
		@@ -1415,15 +1600,10 @@ hook_clone_at_parent(unsigned long flags, void* child_stack, int* ptid,
		static_cast<long>(newtls), 0};
		#endif

		pthread_once(&key_once_control, make_key);
		pthread_setspecific(
		reentrance_guard_key,
		reinterpret_cast<void*>(1)); // cppcheck-suppress intToPointerCast

		in_gkfs_hook = true;
		LOG(SYSCALL, ::get_current_syscall_info() \| gkfs::syscall::executed,
		SYS_clone, args, returned_pid);

		pthread_setspecific(reentrance_guard_key, NULL);
		in_gkfs_hook = false;
		}

		} // namespace
		@@ -1502,22 +1682,17 @@ internal_hook_guard_wrapper(long syscall_number, long arg0, long arg1,
		arg3, arg4, arg5);
		#endif

		pthread_once(&key_once_control, make_key);
		if(pthread_getspecific(reentrance_guard_key) != NULL) {
		if(in_gkfs_hook) {
		::save_current_syscall_info(gkfs::syscall::from_internal_code \|
		gkfs::syscall::to_kernel \|
		gkfs::syscall::not_executed);
		return gkfs::syscall::forward_to_kernel;
		}

		int was_hooked = 0;

		pthread_setspecific(
		reentrance_guard_key,
		reinterpret_cast<void*>(1)); // cppcheck-suppress intToPointerCast
		was_hooked = hook_internal(syscall_number, arg0, arg1, arg2, arg3, arg4,
		in_gkfs_hook = true;
		int was_hooked = hook_internal(syscall_number, arg0, arg1, arg2, arg3, arg4,
		arg5, syscall_return_value);
		pthread_setspecific(reentrance_guard_key, NULL);
		in_gkfs_hook = false;

		return was_hooked;
		}
		@@ -1560,18 +1735,14 @@ hook_guard_wrapper(long syscall_number, long arg0, long arg1, long arg2,
		return gkfs::syscall::forward_to_kernel;
		}

		pthread_once(&key_once_control, make_key); // Ensure the key is created
		if(pthread_getspecific(reentrance_guard_key) != NULL) {
		if(in_gkfs_hook) {
		// While handling an application syscall, nested syscalls originate
		// from the GekkoFS runtime itself. Route them through hook_internal()
		// so native runtime descriptors remain tracked and protected.
		return hook_internal(syscall_number, arg0, arg1, arg2, arg3, arg4, arg5,
		syscall_return_value);
		}
		// Set the guard to a non-NULL value.
		pthread_setspecific(
		reentrance_guard_key,
		reinterpret_cast<void*>(1)); // cppcheck-suppress intToPointerCast
		in_gkfs_hook = true;

		#ifdef SYS_socketcall
		if(syscall_number == SYS_socketcall)
		@@ -1584,8 +1755,7 @@ hook_guard_wrapper(long syscall_number, long arg0, long arg1, long arg2,
		was_hooked = ::hook(syscall_number, arg0, arg1, arg2, arg3, arg4, arg5,
		syscall_return_value);

		// Unset the guard before returning
		pthread_setspecific(reentrance_guard_key, NULL);
		in_gkfs_hook = false;

		return was_hooked;
		}

src/client/open_file_map.cpp

+2 −2

Changes for src/client/open_file_map.cpp: 2 added lines, 2 removed lines.
Original line number	Diff line number	Diff line
		@@ -155,7 +155,7 @@ OpenFileMap::get_shard(int fd) {

		shared_ptr<OpenFile>
		OpenFileMap::get(int fd) {
		if(fd < 0)
		if(fd < 0 \|\| total_files_.load(std::memory_order_relaxed) == 0)
		return nullptr;
		auto& shard = get_shard(fd);
		lock_guard<recursive_mutex> lock(shard.mutex);
		@@ -178,7 +178,7 @@ OpenFileMap::get_dir(int dirfd) {

		bool
		OpenFileMap::exist(const int fd) {
		if(fd < 0)
		if(fd < 0 \|\| total_files_.load(std::memory_order_relaxed) == 0)
		return false;
		auto& shard = get_shard(fd);
		lock_guard<recursive_mutex> lock(shard.mutex);

src/client/preload_context.cpp

+69 −0

Changes for src/client/preload_context.cpp: 69 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -397,6 +397,11 @@ PreloadContext::mountdir(const std::string& path) {
		assert(!gkfs::path::has_trailing_slash(path));
		mountdir_components_ = gkfs::path::split_path(path);
		mountdir_ = path;
		cwd_is_internal_ = (!mountdir_.empty() &&
		(cwd_ == mountdir_ \|\|
		(cwd_.size() > mountdir_.size() &&
		cwd_.compare(0, mountdir_.size(), mountdir_) == 0 &&
		cwd_[mountdir_.size()] == gkfs::path::separator)));
		{
		std::lock_guard<std::mutex> lock(stat_cache_mutex_);
		stat_cache_.clear();
		@@ -425,6 +430,11 @@ PreloadContext::mountdir_components() const {
		void
		PreloadContext::cwd(const std::string& path) {
		cwd_ = path;
		cwd_is_internal_ = (!mountdir_.empty() &&
		(cwd_ == mountdir_ \|\|
		(cwd_.size() > mountdir_.size() &&
		cwd_.compare(0, mountdir_.size(), mountdir_) == 0 &&
		cwd_[mountdir_.size()] == gkfs::path::separator)));
		}

		const std::string&
		@@ -432,6 +442,11 @@ PreloadContext::cwd() const {
		return cwd_;
		}

		bool
		PreloadContext::cwd_is_internal() const noexcept {
		return cwd_is_internal_;
		}

		const thallium::endpoint&
		HostEndpointTable::at(std::size_t id) const {
		if(id >= endpoints_.size()) {
		@@ -709,6 +724,37 @@ PreloadContext::relativize_fd_path(int dirfd, const char* raw_path,
		return RelativizeStatus::fd_unknown;
		}

		// Fast path for non-GekkoFS paths: if absolute and does not match mountdir
		// prefix, or if relative to an external fd / non-internal CWD without
		// parent traversal.
		if(raw_path[0] == gkfs::path::separator) {
		const size_t mlen = mountdir_.size();
		if(strncmp(raw_path, mountdir_.c_str(), mlen) != 0) {
		if(strstr(raw_path, "..") == nullptr) {
		return RelativizeStatus::external;
		}
		} else {
		char next = raw_path[mlen];
		if(next != '\0' && next != gkfs::path::separator) {
		if(strstr(raw_path, "..") == nullptr) {
		return RelativizeStatus::external;
		}
		}
		}
		} else {
		if(dirfd != AT_FDCWD && dirfd >= 0) {
		if(ofm_ && !ofm_->exist(dirfd)) {
		if(strstr(raw_path, "..") == nullptr) {
		return RelativizeStatus::fd_unknown;
		}
		}
		} else if(dirfd == AT_FDCWD && !cwd_is_internal_) {
		if(strstr(raw_path, "..") == nullptr) {
		return RelativizeStatus::external;
		}
		}
		}

		// Skips paths that are used on locking places (i.e. java
		// /proc/sys/vm/overcommit_memory) and produces deadlocks as we call malloc
		// inside.
		@@ -784,6 +830,29 @@ PreloadContext::relativize_path(const char* raw_path,

		// We assume raw path is valid
		assert(raw_path != nullptr);
		if(raw_path == nullptr) {
		return false;
		}

		if(raw_path[0] == gkfs::path::separator) {
		const size_t mlen = mountdir_.size();
		if(strncmp(raw_path, mountdir_.c_str(), mlen) != 0) {
		if(strstr(raw_path, "..") == nullptr) {
		return false;
		}
		} else {
		char next = raw_path[mlen];
		if(next != '\0' && next != gkfs::path::separator) {
		if(strstr(raw_path, "..") == nullptr) {
		return false;
		}
		}
		}
		} else if(!cwd_is_internal_) {
		if(strstr(raw_path, "..") == nullptr) {
		return false;
		}
		}

		std::string path;

include/client/logging.hpp

+2 −2

File changed.

Contains only whitespace changes.

src/client/path.cpp

+2 −2

File changed.

Contains only whitespace changes.