From 6cbdc20c8728823afc3e42055970bc1f4670035b Mon Sep 17 00:00:00 2001 From: rnou Date: Tue, 10 Oct 2023 12:38:22 +0200 Subject: [PATCH 01/14] Creating Gekko API example --- examples/gfind/CMakeLists.txt | 8 +++ examples/gfind/gkfs_lib_example.cpp | 102 ++++++++++++++++++++++++++++ include/client/gkfs_functions.hpp | 3 + include/client/preload.hpp | 5 ++ src/client/gkfs_functions.cpp | 21 ++++++ src/client/hooks.cpp | 13 +--- src/client/preload.cpp | 42 ++++++++++-- 7 files changed, 179 insertions(+), 15 deletions(-) create mode 100644 examples/gfind/gkfs_lib_example.cpp diff --git a/examples/gfind/CMakeLists.txt b/examples/gfind/CMakeLists.txt index d9e63e317..d46441f08 100644 --- a/examples/gfind/CMakeLists.txt +++ b/examples/gfind/CMakeLists.txt @@ -28,9 +28,17 @@ set (CMAKE_CXX_STANDARD 14) add_executable(sfind sfind.cpp) +add_executable(gkfs_lib_example gkfs_lib_example.cpp) + +target_link_libraries(gkfs_lib_example + PRIVATE gkfs_intercept +) if(GKFS_INSTALL_TESTS) install(TARGETS sfind DESTINATION ${CMAKE_INSTALL_BINDIR} ) + install(TARGETS gkfs_lib_example + DESTINATION ${CMAKE_INSTALL_BINDIR} + ) endif() diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/gfind/gkfs_lib_example.cpp new file mode 100644 index 000000000..b1ced51c3 --- /dev/null +++ b/examples/gfind/gkfs_lib_example.cpp @@ -0,0 +1,102 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +/* Function exported from GekkoFS LD_PRELOAD, code needs to be compiled with + * -fPIC */ +extern "C" int gkfs_init() + __attribute__((weak)); + +extern "C" int gkfs_end() + __attribute__((weak)); + +void init_preload() {}; +void destroy_preload() {}; + + + +void write_file(std::string filename){ + // Opem File + int fd = gkfs::syscall::gkfs_open(filename,S_IRWXU,O_RDWR|O_CREAT); + + cout << "FD open " << fd << endl; + char *buf = "testing"; + + // int size = gkfs::syscall::gkfs_write(fd, buf, 7); + + // cout << "FD size" << size << endl; + + gkfs::syscall::gkfs_close(fd); + +} + + +void read_file(std::string filename){ + int fdread = gkfs::syscall::gkfs_open(filename,S_IRWXU,O_RDONLY); + char *bufread = "TESTING\0"; + int sizeread = gkfs::syscall::gkfs_read(fdread, bufread, 7); + + cout << "Reading : " << sizeread << " --> " << bufread << endl; + + gkfs::syscall::gkfs_close(fdread); + +} +int main(int argc, char **argv) { + cout << "GekkoFS Client library test" << endl; + + auto res = gkfs_init(); + + cout << "Init result " << res << endl; + + //write_file ("/test.tmp"); + + read_file("/test.tmp"); + + res = gkfs_end(); + + cout << "End result " << res << endl; + +} diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 63b9b655f..574b084ee 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -149,6 +149,9 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, int gkfs_rmdir(const std::string& path); +int +gkfs_close(unsigned int fd); + #ifdef HAS_RENAME int gkfs_rename(const std::string& old_path, const std::string& new_path); diff --git a/include/client/preload.hpp b/include/client/preload.hpp index 6cae014b1..929e30eaf 100644 --- a/include/client/preload.hpp +++ b/include/client/preload.hpp @@ -47,5 +47,10 @@ init_preload() __attribute__((constructor)); void destroy_preload() __attribute__((destructor)); +extern "C" int +gkfs_init(); + +extern "C" int +gkfs_end(); #endif // IOINTERCEPT_PRELOAD_HPP diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index b03e8eaeb..ef63cd8d0 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1394,6 +1394,27 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, return written; } +/** + * @brief Closes an fd. To be used externally + * + * @param fd + * @return int + */ +int gkfs_close(unsigned int fd) { + if(CTX->file_map()->exist(fd)) { + // No call to the daemon is required + CTX->file_map()->remove(fd); + return 0; + } + + if(CTX->is_internal_fd(fd)) { + // the client application (for some reason) is trying to close an + // internal fd: ignore it + return 0; + } + + return -1; +} #ifdef HAS_SYMLINKS #ifdef GKFS_ENABLE_UNUSED_FUNCTIONS diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index a986d5898..ea7559c46 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -91,17 +91,10 @@ hook_close(int fd) { LOG(DEBUG, "{}() called with fd: {}", __func__, fd); - if(CTX->file_map()->exist(fd)) { - // No call to the daemon is required - CTX->file_map()->remove(fd); + auto ret = gkfs::syscall::gkfs_close(fd); + + if (ret == 0) return 0; - } - - if(CTX->is_internal_fd(fd)) { - // the client application (for some reason) is trying to close an - // internal fd: ignore it - return 0; - } return syscall_no_intercept_wrapper(SYS_close, fd); } diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 6d7a0c0e9..589656796 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -67,8 +67,10 @@ exit_error_msg(int errcode, const string& msg) { // if we don't disable interception before calling ::exit() // syscall hooks may find an inconsistent in shared state // (e.g. the logger) and thus, crash - gkfs::preload::stop_interception(); - CTX->disable_interception(); + if(CTX->interception_enabled()) { + gkfs::preload::stop_interception(); + CTX->disable_interception(); + } ::exit(errcode); } @@ -314,9 +316,39 @@ destroy_preload() { ld_network_service.reset(); LOG(DEBUG, "RPC subsystem shut down"); - gkfs::preload::stop_interception(); - CTX->disable_interception(); - LOG(DEBUG, "Syscall interception stopped"); + if(CTX->interception_enabled()) { + gkfs::preload::stop_interception(); + CTX->disable_interception(); + LOG(DEBUG, "Syscall interception stopped"); + } LOG(INFO, "All subsystems shut down. Client shutdown complete."); } + + + +/** + * @brief External functions to call linking the library + * + */ +extern "C" int +gkfs_init() { + CTX->init_logging(); + // from here ownwards it is safe to print messages + LOG(DEBUG, "Logging subsystem initialized"); + gkfs::preload::init_environment(); + return 4; +} + + +extern "C" int +gkfs_end() { + CTX->clear_hosts(); + LOG(DEBUG, "Peer information deleted"); + + ld_network_service.reset(); + LOG(DEBUG, "RPC subsystem shut down"); + + LOG(INFO, "All subsystems shut down. Client shutdown complete."); + return 4; +} \ No newline at end of file -- GitLab From 9f5e4f349420e8c2d0e8a099dfc6002bc040aa6b Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 14 Mar 2024 16:34:04 +0100 Subject: [PATCH 02/14] 2 libs, disable syscall_intercept --- examples/gfind/CMakeLists.txt | 2 +- examples/gfind/gkfs_lib_example.cpp | 2 -- src/client/CMakeLists.txt | 38 +++++++++++++++++++++ src/client/void_syscall_intercept.cpp | 49 +++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 src/client/void_syscall_intercept.cpp diff --git a/examples/gfind/CMakeLists.txt b/examples/gfind/CMakeLists.txt index d46441f08..22c9bfa99 100644 --- a/examples/gfind/CMakeLists.txt +++ b/examples/gfind/CMakeLists.txt @@ -31,7 +31,7 @@ add_executable(sfind sfind.cpp) add_executable(gkfs_lib_example gkfs_lib_example.cpp) target_link_libraries(gkfs_lib_example - PRIVATE gkfs_intercept + PRIVATE gkfs_user_lib ) if(GKFS_INSTALL_TESTS) diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/gfind/gkfs_lib_example.cpp index b1ced51c3..3ed6e301b 100644 --- a/examples/gfind/gkfs_lib_example.cpp +++ b/examples/gfind/gkfs_lib_example.cpp @@ -56,8 +56,6 @@ extern "C" int gkfs_end() void init_preload() {}; void destroy_preload() {}; - - void write_file(std::string filename){ // Opem File int fd = gkfs::syscall::gkfs_open(filename,S_IRWXU,O_RDWR|O_CREAT); diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index c408fc726..7961f8464 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -32,6 +32,7 @@ # based on syscall interception. # ############################################################################## add_library(gkfs_intercept SHARED) +add_library (gkfs_user_lib SHARED) target_sources( gkfs_intercept @@ -52,6 +53,25 @@ target_sources( syscalls/detail/syscall_info.c ) +target_sources( + gkfs_user_lib + PRIVATE void_syscall_intercept.cpp + gkfs_functions.cpp + hooks.cpp + intercept.cpp + logging.cpp + open_file_map.cpp + open_dir.cpp + path.cpp + preload.cpp + preload_context.cpp + preload_util.cpp + rpc/rpc_types.cpp + rpc/forward_data.cpp + rpc/forward_management.cpp + rpc/forward_metadata.cpp + syscalls/detail/syscall_info.c +) if(GKFS_ENABLE_AGIOS) target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) @@ -69,6 +89,16 @@ target_link_libraries( Syscall_intercept::Syscall_intercept ) +target_link_libraries( + gkfs_user_lib + PRIVATE metadata distributor env_util arithmetic path_util rpc_utils + PUBLIC dl + Mercury::Mercury + hermes + fmt::fmt + Threads::Threads +) + install( TARGETS gkfs_intercept LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} @@ -76,3 +106,11 @@ install( PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) + +install( + TARGETS gkfs_user_lib + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs +) + diff --git a/src/client/void_syscall_intercept.cpp b/src/client/void_syscall_intercept.cpp new file mode 100644 index 000000000..744dd7195 --- /dev/null +++ b/src/client/void_syscall_intercept.cpp @@ -0,0 +1,49 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS' POSIX interface. + + GekkoFS' POSIX interface is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GekkoFS' POSIX interface is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with GekkoFS' POSIX interface. If not, see + . + + SPDX-License-Identifier: LGPL-3.0-or-later +*/ +extern "C" { +int (*intercept_hook_point)(long syscall_number, + long arg0, long arg1, + long arg2, long arg3, + long arg4, long arg5, + long *result) {}; +void (*intercept_hook_point_clone_child)( + unsigned long flags, void *child_stack, + int *ptid, int *ctid, long newtls) {}; +void (*intercept_hook_point_clone_parent)( + unsigned long flags, void *child_stack, + int *ptid, int *ctid, long newtls, + long returned_pid) {}; + void (*intercept_hook_point_post_kernel)(long syscall_number, + long arg0, long arg1, + long arg2, long arg3, + long arg4, long arg5, + long result) {}; + +long syscall_no_intercept(long syscall_number, ...) { return 0}; +} \ No newline at end of file -- GitLab From 395e9329bf297f743687405957cbc0c0c4f1d25e Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 11 Oct 2023 09:45:53 +0200 Subject: [PATCH 03/14] Library finish --- CMakeLists.txt | 13 ++-- examples/gfind/gkfs_lib_example.cpp | 66 ++++++++++--------- include/client/CMakeLists.txt | 26 ++++++++ include/client/gkfs_functions.hpp | 1 + include/client/hooks.hpp | 6 ++ include/client/logging.hpp | 4 ++ .../client/void_syscall_intercept.hpp | 36 +++++----- src/client/CMakeLists.txt | 34 +++++----- src/client/gkfs_functions.cpp | 11 ++-- src/client/hooks.cpp | 4 +- src/client/intercept.cpp | 18 +++++ src/client/path.cpp | 7 +- src/client/preload.cpp | 9 ++- src/client/preload_context.cpp | 8 ++- src/client/syscalls/util.S | 49 ++++++++++++++ 15 files changed, 202 insertions(+), 90 deletions(-) rename src/client/void_syscall_intercept.cpp => include/client/void_syscall_intercept.hpp (64%) create mode 100644 src/client/syscalls/util.S diff --git a/CMakeLists.txt b/CMakeLists.txt index c70d36115..cda0d59e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,16 +32,17 @@ cmake_minimum_required(VERSION 3.13) project( GekkoFS VERSION 0.9.2 + LANGUAGES ASM CXX C ) enable_testing() -if (NOT CMAKE_COMPILER_IS_GNUCC) - message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") -endif () -if (NOT CMAKE_COMPILER_IS_GNUCXX) - message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported") -endif () +#if (NOT CMAKE_COMPILER_IS_GNUCC) + # message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") +#endif () +#if (NOT CMAKE_COMPILER_IS_GNUCXX) +# message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported") +#endif () set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/gfind/gkfs_lib_example.cpp index 3ed6e301b..513b0ef98 100644 --- a/examples/gfind/gkfs_lib_example.cpp +++ b/examples/gfind/gkfs_lib_example.cpp @@ -47,54 +47,58 @@ using namespace std; /* Function exported from GekkoFS LD_PRELOAD, code needs to be compiled with * -fPIC */ -extern "C" int gkfs_init() - __attribute__((weak)); +extern "C" int +gkfs_init() __attribute__((weak)); -extern "C" int gkfs_end() - __attribute__((weak)); +extern "C" int +gkfs_end() __attribute__((weak)); -void init_preload() {}; -void destroy_preload() {}; +void +init_preload(){}; +void +destroy_preload(){}; -void write_file(std::string filename){ - // Opem File - int fd = gkfs::syscall::gkfs_open(filename,S_IRWXU,O_RDWR|O_CREAT); +void +write_file(std::string filename) { + // Open File + int fd = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDWR | O_CREAT); - cout << "FD open " << fd << endl; - char *buf = "testing"; - - // int size = gkfs::syscall::gkfs_write(fd, buf, 7); + cout << "FD open " << fd << endl; + char* buf = "testing"; - // cout << "FD size" << size << endl; + int size = gkfs::syscall::gkfs_write(fd, buf, 7); - gkfs::syscall::gkfs_close(fd); + cout << "FD size" << size << endl; + gkfs::syscall::gkfs_close(fd); } -void read_file(std::string filename){ - int fdread = gkfs::syscall::gkfs_open(filename,S_IRWXU,O_RDONLY); - char *bufread = "TESTING\0"; - int sizeread = gkfs::syscall::gkfs_read(fdread, bufread, 7); +void +read_file(std::string filename) { + int fdread = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDONLY); + if(fdread == -1) + return; + char* bufread = (char*) malloc(10); + int sizeread = gkfs::syscall::gkfs_read(fdread, bufread, 7); - cout << "Reading : " << sizeread << " --> " << bufread << endl; - - gkfs::syscall::gkfs_close(fdread); + cout << "Reading : " << sizeread << " --> " << bufread << endl; + gkfs::syscall::gkfs_close(fdread); } -int main(int argc, char **argv) { - cout << "GekkoFS Client library test" << endl; +int +main(int argc, char** argv) { + cout << "GekkoFS Client library test" << endl; - auto res = gkfs_init(); + auto res = gkfs_init(); - cout << "Init result " << res << endl; + cout << "Init result " << res << endl; - //write_file ("/test.tmp"); - - read_file("/test.tmp"); + write_file("/test.tmp"); - res = gkfs_end(); + read_file("/test.tmp"); - cout << "End result " << res << endl; + res = gkfs_end(); + cout << "End result " << res << endl; } diff --git a/include/client/CMakeLists.txt b/include/client/CMakeLists.txt index 2d12db14e..b82b0b96a 100644 --- a/include/client/CMakeLists.txt +++ b/include/client/CMakeLists.txt @@ -54,3 +54,29 @@ target_sources( syscalls/detail/syscall_info.h ) +target_sources( + gkfs_user_lib + PUBLIC gkfs_functions.hpp + env.hpp + hooks.hpp + intercept.hpp + logging.hpp + make_array.hpp + open_file_map.hpp + open_dir.hpp + path.hpp + preload.hpp + preload_context.hpp + preload_util.hpp + rpc/rpc_types.hpp + rpc/forward_management.hpp + rpc/forward_metadata.hpp + rpc/forward_data.hpp + syscalls/args.hpp + syscalls/decoder.hpp + syscalls/errno.hpp + syscalls/rets.hpp + syscalls/syscall.hpp + syscalls/detail/syscall_info.h + void_syscall_intercept.hpp +) diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 574b084ee..60cbb05a5 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -156,6 +156,7 @@ gkfs_close(unsigned int fd); int gkfs_rename(const std::string& old_path, const std::string& new_path); #endif // HAS_RENAME + } // namespace gkfs::syscall // gkfs_getsingleserverdir is using extern "C" to demangle it for C usage diff --git a/include/client/hooks.hpp b/include/client/hooks.hpp index e71a6a08d..fa9ec424b 100644 --- a/include/client/hooks.hpp +++ b/include/client/hooks.hpp @@ -36,7 +36,13 @@ extern "C" { #include #include } + +#ifndef BYPASS_SYSCALL #include +#else +#include +#endif + /* * For PowerPC, syscall_no_intercept_wrapper() is defined in the diff --git a/include/client/logging.hpp b/include/client/logging.hpp index 2f00058d0..9d1f464e5 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -30,7 +30,11 @@ #ifndef LIBGKFS_LOGGING_HPP #define LIBGKFS_LOGGING_HPP +#ifndef BYPASS_SYSCALL #include +#else +#include +#endif #include #include diff --git a/src/client/void_syscall_intercept.cpp b/include/client/void_syscall_intercept.hpp similarity index 64% rename from src/client/void_syscall_intercept.cpp rename to include/client/void_syscall_intercept.hpp index 744dd7195..16cd01b5b 100644 --- a/src/client/void_syscall_intercept.cpp +++ b/include/client/void_syscall_intercept.hpp @@ -26,24 +26,20 @@ SPDX-License-Identifier: LGPL-3.0-or-later */ +#ifndef SYSCALL_BYPASS_HPP +#define SYSCALL_BYPASS_HPP + +static inline int +syscall_error_code(long result) { + if(result < 0 && result >= -0x1000) + return (int) -result; + + return 0; +} + extern "C" { -int (*intercept_hook_point)(long syscall_number, - long arg0, long arg1, - long arg2, long arg3, - long arg4, long arg5, - long *result) {}; -void (*intercept_hook_point_clone_child)( - unsigned long flags, void *child_stack, - int *ptid, int *ctid, long newtls) {}; -void (*intercept_hook_point_clone_parent)( - unsigned long flags, void *child_stack, - int *ptid, int *ctid, long newtls, - long returned_pid) {}; - void (*intercept_hook_point_post_kernel)(long syscall_number, - long arg0, long arg1, - long arg2, long arg3, - long arg4, long arg5, - long result) {}; - -long syscall_no_intercept(long syscall_number, ...) { return 0}; -} \ No newline at end of file +long +syscall_no_intercept(long syscall_number, ...); +} + +#endif \ No newline at end of file diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 7961f8464..afd514c0b 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -31,14 +31,15 @@ # This builds the `libgkfs_intercept.so` library: the primary GekkoFS client # based on syscall interception. # ############################################################################## -add_library(gkfs_intercept SHARED) + +add_library (gkfs_intercept SHARED) add_library (gkfs_user_lib SHARED) -target_sources( - gkfs_intercept - PRIVATE gkfs_functions.cpp - hooks.cpp + +target_sources(gkfs_intercept + PRIVATE gkfs_functions.cpp intercept.cpp + hooks.cpp logging.cpp open_file_map.cpp open_dir.cpp @@ -50,15 +51,13 @@ target_sources( rpc/forward_data.cpp rpc/forward_management.cpp rpc/forward_metadata.cpp - syscalls/detail/syscall_info.c -) + syscalls/detail/syscall_info.c) target_sources( gkfs_user_lib - PRIVATE void_syscall_intercept.cpp - gkfs_functions.cpp - hooks.cpp + PRIVATE gkfs_functions.cpp intercept.cpp + hooks.cpp logging.cpp open_file_map.cpp open_dir.cpp @@ -70,18 +69,19 @@ target_sources( rpc/forward_data.cpp rpc/forward_management.cpp rpc/forward_metadata.cpp - syscalls/detail/syscall_info.c + syscalls/detail/syscall_info.c syscalls/util.S ) - if(GKFS_ENABLE_AGIOS) - target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) - endif() +target_compile_definitions(gkfs_user_lib PUBLIC BYPASS_SYSCALL) + +if(GKFS_ENABLE_AGIOS) + target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) +endif() target_link_libraries( gkfs_intercept PRIVATE metadata distributor env_util arithmetic path_util rpc_utils - PUBLIC Syscall_intercept::Syscall_intercept - dl + PUBLIC dl Mercury::Mercury hermes fmt::fmt @@ -106,11 +106,9 @@ install( PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) - install( TARGETS gkfs_user_lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) - diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index ef63cd8d0..ea87d44a8 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1396,11 +1396,12 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, /** * @brief Closes an fd. To be used externally - * - * @param fd - * @return int + * + * @param fd + * @return int */ -int gkfs_close(unsigned int fd) { +int +gkfs_close(unsigned int fd) { if(CTX->file_map()->exist(fd)) { // No call to the daemon is required CTX->file_map()->remove(fd); @@ -1412,7 +1413,7 @@ int gkfs_close(unsigned int fd) { // internal fd: ignore it return 0; } - + return -1; } diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index ea7559c46..9a54048c5 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -92,8 +92,8 @@ hook_close(int fd) { LOG(DEBUG, "{}() called with fd: {}", __func__, fd); auto ret = gkfs::syscall::gkfs_close(fd); - - if (ret == 0) + + if(ret == 0) return 0; return syscall_no_intercept_wrapper(SYS_close, fd); diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index 316c5e398..7a8a455fe 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -44,6 +44,24 @@ extern "C" { #include } + +#ifdef BYPASS_SYSCALL +int (*intercept_hook_point)(long syscall_number, long arg0, long arg1, + long arg2, long arg3, long arg4, long arg5, + long* result){}; + +void (*intercept_hook_point_clone_child)(unsigned long flags, void* child_stack, + int* ptid, int* ctid, long newtls){}; + +void (*intercept_hook_point_clone_parent)(unsigned long flags, + void* child_stack, int* ptid, + int* ctid, long newtls, + long returned_pid){}; + +void (*intercept_hook_point_post_kernel)(long syscall_number, long arg0, + long arg1, long arg2, long arg3, + long arg4, long arg5, long result){}; +#endif namespace { thread_local bool reentrance_guard_flag; diff --git a/src/client/path.cpp b/src/client/path.cpp index 4f72b4807..85357209a 100644 --- a/src/client/path.cpp +++ b/src/client/path.cpp @@ -39,9 +39,14 @@ #include #include +#ifndef BYPASS_SYSCALL +#include +#else +#include +#endif + extern "C" { #include -#include } using namespace std; diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 589656796..b989eeb6a 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -67,7 +67,7 @@ exit_error_msg(int errcode, const string& msg) { // if we don't disable interception before calling ::exit() // syscall hooks may find an inconsistent in shared state // (e.g. the logger) and thus, crash - if(CTX->interception_enabled()) { + if(CTX->interception_enabled()) { gkfs::preload::stop_interception(); CTX->disable_interception(); } @@ -316,7 +316,7 @@ destroy_preload() { ld_network_service.reset(); LOG(DEBUG, "RPC subsystem shut down"); - if(CTX->interception_enabled()) { + if(CTX->interception_enabled()) { gkfs::preload::stop_interception(); CTX->disable_interception(); LOG(DEBUG, "Syscall interception stopped"); @@ -326,10 +326,9 @@ destroy_preload() { } - /** * @brief External functions to call linking the library - * + * */ extern "C" int gkfs_init() { @@ -343,7 +342,7 @@ gkfs_init() { extern "C" int gkfs_end() { - CTX->clear_hosts(); + CTX->clear_hosts(); LOG(DEBUG, "Peer information deleted"); ld_network_service.reset(); diff --git a/src/client/preload_context.cpp b/src/client/preload_context.cpp index 7101041dc..5dbc75c87 100644 --- a/src/client/preload_context.cpp +++ b/src/client/preload_context.cpp @@ -37,13 +37,17 @@ #include #include #include - #include #include -extern "C" { +#ifndef BYPASS_SYSCALL #include +#else +#include +#endif + +extern "C" { #include } diff --git a/src/client/syscalls/util.S b/src/client/syscalls/util.S new file mode 100644 index 000000000..67c9b279a --- /dev/null +++ b/src/client/syscalls/util.S @@ -0,0 +1,49 @@ +/* + * Copyright 2016-2017, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +.global syscall_no_intercept; +.type syscall_no_intercept, @function + +.text + +syscall_no_intercept: + movq %rdi, %rax /* convert from linux ABI calling */ + movq %rsi, %rdi /* convention to syscall calling convention */ + movq %rdx, %rsi + movq %rcx, %rdx + movq %r8, %r10 + movq %r9, %r8 + movq 8(%rsp), %r9 + syscall + ret + +.size syscall_no_intercept, .-syscall_no_intercept -- GitLab From 965614f45006defb28ddd0bf6b0f3836ade66520 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 11 Oct 2023 11:30:14 +0200 Subject: [PATCH 04/14] Create an user_functions.hpp --- CMakeLists.txt | 2 +- examples/gfind/gkfs_lib_example.cpp | 29 +++++-------- include/client/CMakeLists.txt | 1 + include/client/gkfs_functions.hpp | 1 + include/client/preload.hpp | 8 +--- include/client/user_functions.hpp | 67 +++++++++++++++++++++++++++++ src/client/CMakeLists.txt | 7 +++ src/client/preload.cpp | 8 +++- 8 files changed, 96 insertions(+), 27 deletions(-) create mode 100644 include/client/user_functions.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cda0d59e7..a26d68077 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,7 +38,7 @@ project( enable_testing() #if (NOT CMAKE_COMPILER_IS_GNUCC) - # message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") +# message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") #endif () #if (NOT CMAKE_COMPILER_IS_GNUCXX) # message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported") diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/gfind/gkfs_lib_example.cpp index 513b0ef98..a86dd5812 100644 --- a/examples/gfind/gkfs_lib_example.cpp +++ b/examples/gfind/gkfs_lib_example.cpp @@ -41,35 +41,26 @@ #include #include #include -#include +#include using namespace std; -/* Function exported from GekkoFS LD_PRELOAD, code needs to be compiled with - * -fPIC */ -extern "C" int -gkfs_init() __attribute__((weak)); - -extern "C" int -gkfs_end() __attribute__((weak)); - -void -init_preload(){}; -void -destroy_preload(){}; - void write_file(std::string filename) { // Open File int fd = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDWR | O_CREAT); cout << "FD open " << fd << endl; - char* buf = "testing"; - int size = gkfs::syscall::gkfs_write(fd, buf, 7); + char* bufwrite = (char*) malloc(10); + strncpy(bufwrite, "testing", 7); + - cout << "FD size" << size << endl; + int size = gkfs::syscall::gkfs_write(fd, bufwrite, 7); + cout << "Writting size " << size << endl; + + free(bufwrite); gkfs::syscall::gkfs_close(fd); } @@ -82,10 +73,12 @@ read_file(std::string filename) { char* bufread = (char*) malloc(10); int sizeread = gkfs::syscall::gkfs_read(fdread, bufread, 7); - cout << "Reading : " << sizeread << " --> " << bufread << endl; + cout << "Reading Size: " << sizeread << " Content: " << bufread << endl; + free(bufread); gkfs::syscall::gkfs_close(fdread); } + int main(int argc, char** argv) { cout << "GekkoFS Client library test" << endl; diff --git a/include/client/CMakeLists.txt b/include/client/CMakeLists.txt index b82b0b96a..6815fc953 100644 --- a/include/client/CMakeLists.txt +++ b/include/client/CMakeLists.txt @@ -79,4 +79,5 @@ target_sources( syscalls/syscall.hpp syscalls/detail/syscall_info.h void_syscall_intercept.hpp + user_functions.hpp ) diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 60cbb05a5..6e97df983 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -163,4 +163,5 @@ gkfs_rename(const std::string& old_path, const std::string& new_path); extern "C" int gkfs_getsingleserverdir(const char* path, struct dirent_extended* dirp, unsigned int count, int server); + #endif // GEKKOFS_GKFS_FUNCTIONS_HPP diff --git a/include/client/preload.hpp b/include/client/preload.hpp index 929e30eaf..afad97c68 100644 --- a/include/client/preload.hpp +++ b/include/client/preload.hpp @@ -41,16 +41,12 @@ void init_ld_env_if_needed(); } // namespace gkfs::preload +#ifndef BYPASS_SYSCALL void init_preload() __attribute__((constructor)); void destroy_preload() __attribute__((destructor)); - -extern "C" int -gkfs_init(); - -extern "C" int -gkfs_end(); +#endif #endif // IOINTERCEPT_PRELOAD_HPP diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp new file mode 100644 index 000000000..e041ba61a --- /dev/null +++ b/include/client/user_functions.hpp @@ -0,0 +1,67 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS' POSIX interface. + + GekkoFS' POSIX interface is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GekkoFS' POSIX interface is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with GekkoFS' POSIX interface. If not, see + . + + SPDX-License-Identifier: LGPL-3.0-or-later +*/ + +#ifndef GEKKOFS_USER_FUNCTIONS_HPP +#define GEKKOFS_USER_FUNCTIONS_HPP +#include +#include +#include +#include + +namespace gkfs::syscall { + +int +gkfs_open(const std::string& path, mode_t mode, int flags); + +int +gkfs_remove(const std::string& path); + + +ssize_t +gkfs_write(int fd, const void* buf, size_t count); + + +ssize_t +gkfs_read(int fd, void* buf, size_t count); + + +int +gkfs_close(unsigned int fd); + + +} // namespace gkfs::syscall + + +extern "C" int +gkfs_init(); + +extern "C" int +gkfs_end(); + +#endif // GEKKOFS_USER_FUNCTIONS_HPP diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index afd514c0b..6550147da 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -106,6 +106,13 @@ install( PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) + +set_target_properties(gkfs_user_lib + PROPERTIES + PUBLIC_HEADER "../../include/client/void_syscall_intercept.hpp" +) + + install( TARGETS gkfs_user_lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/src/client/preload.cpp b/src/client/preload.cpp index b989eeb6a..652ef3580 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -333,10 +333,13 @@ destroy_preload() { extern "C" int gkfs_init() { CTX->init_logging(); + // from here ownwards it is safe to print messages LOG(DEBUG, "Logging subsystem initialized"); + gkfs::preload::init_environment(); - return 4; + + return 0; } @@ -349,5 +352,6 @@ gkfs_end() { LOG(DEBUG, "RPC subsystem shut down"); LOG(INFO, "All subsystems shut down. Client shutdown complete."); - return 4; + + return 0; } \ No newline at end of file -- GitLab From 3fefaec3fadfbcdba79d1755d06a38fa06bb7924 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 14 Mar 2024 11:53:22 +0100 Subject: [PATCH 05/14] Add headers and new user function headers --- CMake/FindMercury.cmake | 2 +- include/client/user_functions.hpp | 14 +++++++++++--- src/client/CMakeLists.txt | 1 + 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CMake/FindMercury.cmake b/CMake/FindMercury.cmake index 7eb96cde5..451ed6d40 100644 --- a/CMake/FindMercury.cmake +++ b/CMake/FindMercury.cmake @@ -177,7 +177,7 @@ endmacro() include(SelectLibraryConfigurations) -set(_mercury_components na mercury_util mercury_hl) +set(_mercury_components na mercury_util) # prevent repeating work if the main CMakeLists.txt already called # find_package(PkgConfig) diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp index e041ba61a..776e84f3b 100644 --- a/include/client/user_functions.hpp +++ b/include/client/user_functions.hpp @@ -40,20 +40,28 @@ int gkfs_open(const std::string& path, mode_t mode, int flags); int -gkfs_remove(const std::string& path); +gkfs_create(const std::string& path, mode_t mode); +int +gkfs_remove(const std::string& path); ssize_t gkfs_write(int fd, const void* buf, size_t count); - ssize_t gkfs_read(int fd, void* buf, size_t count); - int gkfs_close(unsigned int fd); +off64_t +gkfs_lseek(unsigned int fd, off64_t offset, unsigned int whence); + +ssize_t +gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset); + +ssize_t +gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset); } // namespace gkfs::syscall diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 6550147da..a4827af37 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -110,6 +110,7 @@ install( set_target_properties(gkfs_user_lib PROPERTIES PUBLIC_HEADER "../../include/client/void_syscall_intercept.hpp" + PUBLIC_HEADER "../../include/client/user_functions.hpp" ) -- GitLab From 649df72d173dc25004cdd9698b12aa60552a7a5c Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 28 Feb 2024 17:13:01 +0100 Subject: [PATCH 06/14] Adding gkfs_stat to the user lib --- include/client/user_functions.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp index 776e84f3b..c26e39d43 100644 --- a/include/client/user_functions.hpp +++ b/include/client/user_functions.hpp @@ -34,6 +34,8 @@ #include #include +struct linux_dirent64; + namespace gkfs::syscall { int @@ -63,6 +65,9 @@ gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset); ssize_t gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset); +int +gkfs_stat(const std::string& path, struct stat* buf, bool follow_links = true); + } // namespace gkfs::syscall -- GitLab From 751bceaad7dd205340c864f4264a8b40d87a7cd1 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 29 Feb 2024 13:07:10 +0100 Subject: [PATCH 07/14] Add file_list function, remove and expand gkfs_lib example --- examples/gfind/gkfs_lib_example.cpp | 18 ++++++++++++++++++ include/client/gkfs_functions.hpp | 3 +++ include/client/user_functions.hpp | 5 +++++ src/client/gkfs_functions.cpp | 22 ++++++++++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/gfind/gkfs_lib_example.cpp index a86dd5812..4644d29fc 100644 --- a/examples/gfind/gkfs_lib_example.cpp +++ b/examples/gfind/gkfs_lib_example.cpp @@ -91,6 +91,24 @@ main(int argc, char** argv) { read_file("/test.tmp"); + + write_file("/secondfile.tmp"); + + auto f_list = gkfs::syscall::gkfs_get_file_list("/"); + + for(auto f : f_list) { + cout << "File: " << f << endl; + struct stat buf; + memset(&buf, 0, sizeof(struct stat)); + + gkfs::syscall::gkfs_stat("/" + f, &buf, true); + + cout << "Size: " << buf.st_size << " Mode: " << buf.st_mode << endl; + cout << "Atime: " << buf.st_atime << " Mtime: " << buf.st_mtime + << " Ctime: " << buf.st_ctime << endl + << " ****** " << endl; + } + res = gkfs_end(); cout << "End result " << res << endl; diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 6e97df983..313df5fcb 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -152,6 +152,9 @@ gkfs_rmdir(const std::string& path); int gkfs_close(unsigned int fd); +std::vector +gkfs_get_file_list(const std::string& path); + #ifdef HAS_RENAME int gkfs_rename(const std::string& old_path, const std::string& new_path); diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp index c26e39d43..5d1bfdfe4 100644 --- a/include/client/user_functions.hpp +++ b/include/client/user_functions.hpp @@ -68,6 +68,11 @@ gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset); int gkfs_stat(const std::string& path, struct stat* buf, bool follow_links = true); +int +gkfs_remove(const std::string& path); + +std::vector +gkfs_get_file_list(const std::string& path); } // namespace gkfs::syscall diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index ea87d44a8..865d38214 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1504,6 +1504,28 @@ gkfs_readlink(const std::string& path, char* buf, int bufsize) { #endif #endif + +std::vector +gkfs_get_file_list(const std::string& path) { + auto ret = gkfs::rpc::forward_get_dirents(path); + auto err = ret.first; + if(err) { + errno = err; + return {}; + } + + auto open_dir = ret.second; + + std::vector file_list; + unsigned int pos = 0; + + while(pos < open_dir->size()) { + auto de = open_dir->getdent(pos++); + file_list.push_back(de.name()); + } + return file_list; +} + } // namespace gkfs::syscall -- GitLab From b749fdf9bea637c71a9664acfdde54e538eb7e0c Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 6 Jun 2024 07:21:17 +0200 Subject: [PATCH 08/14] Gfind CMake updated to Cxx 17 --- examples/gfind/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/gfind/CMakeLists.txt b/examples/gfind/CMakeLists.txt index 22c9bfa99..1bf8c698d 100644 --- a/examples/gfind/CMakeLists.txt +++ b/examples/gfind/CMakeLists.txt @@ -26,7 +26,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ -set (CMAKE_CXX_STANDARD 14) +set (CMAKE_CXX_STANDARD 17) add_executable(sfind sfind.cpp) add_executable(gkfs_lib_example gkfs_lib_example.cpp) -- GitLab From 634f0a7149e9362a972f2ed7da7efcbb32030e50 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Sat, 8 Jun 2024 10:16:16 +0200 Subject: [PATCH 09/14] Update Hermes for CMake fixes --- external/hermes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/hermes b/external/hermes index 2fa83ff47..98272ebc2 160000 --- a/external/hermes +++ b/external/hermes @@ -1 +1 @@ -Subproject commit 2fa83ff4787d60e2d849c227e64d1b691f8fe0e4 +Subproject commit 98272ebc24405f556b0c0d23405087976d817c7a -- GitLab From e70867a12072e958e5d302539107cf99d3194723 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Sat, 8 Jun 2024 10:41:27 +0200 Subject: [PATCH 10/14] CMake cleanup --- CMakeLists.txt | 14 +++---- examples/CMakeLists.txt | 30 ++++++++++++++ examples/gfind/CMakeLists.txt | 12 +----- examples/user_library/CMakeLists.txt | 39 +++++++++++++++++++ .../gkfs_lib_example.cpp | 0 5 files changed, 78 insertions(+), 17 deletions(-) create mode 100644 examples/CMakeLists.txt create mode 100644 examples/user_library/CMakeLists.txt rename examples/{gfind => user_library}/gkfs_lib_example.cpp (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index a26d68077..962878eee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,12 +37,12 @@ project( enable_testing() -#if (NOT CMAKE_COMPILER_IS_GNUCC) -# message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") -#endif () -#if (NOT CMAKE_COMPILER_IS_GNUCXX) -# message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported") -#endif () +if (NOT CMAKE_COMPILER_IS_GNUCC) + message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") +endif () +if (NOT CMAKE_COMPILER_IS_GNUCXX) + message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported") +endif () set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -346,7 +346,7 @@ if (GKFS_BUILD_TESTS) message(STATUS "[gekkofs] Guided distributor tests: ${GKFS_TESTS_GUIDED_DISTRIBUTION}") add_subdirectory(tests) - add_subdirectory(examples/gfind) + add_subdirectory(examples) else() unset(GKFS_TESTS_INTERFACE CACHE) endif() diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 000000000..f8a4ad5ed --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,30 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +add_subdirectory(gfind) +add_subdirectory(user_library) \ No newline at end of file diff --git a/examples/gfind/CMakeLists.txt b/examples/gfind/CMakeLists.txt index 1bf8c698d..fef70119d 100644 --- a/examples/gfind/CMakeLists.txt +++ b/examples/gfind/CMakeLists.txt @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -28,17 +28,9 @@ set (CMAKE_CXX_STANDARD 17) add_executable(sfind sfind.cpp) -add_executable(gkfs_lib_example gkfs_lib_example.cpp) - -target_link_libraries(gkfs_lib_example - PRIVATE gkfs_user_lib -) if(GKFS_INSTALL_TESTS) install(TARGETS sfind DESTINATION ${CMAKE_INSTALL_BINDIR} ) - install(TARGETS gkfs_lib_example - DESTINATION ${CMAKE_INSTALL_BINDIR} - ) endif() diff --git a/examples/user_library/CMakeLists.txt b/examples/user_library/CMakeLists.txt new file mode 100644 index 000000000..ee8d915ab --- /dev/null +++ b/examples/user_library/CMakeLists.txt @@ -0,0 +1,39 @@ +################################################################################ +# Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +set(CMAKE_CXX_STANDARD 17) +add_executable(gkfs_lib_example gkfs_lib_example.cpp) + +target_link_libraries(gkfs_lib_example + PRIVATE gkfs_user_lib +) +if (GKFS_INSTALL_TESTS) + install(TARGETS gkfs_lib_example + DESTINATION ${CMAKE_INSTALL_BINDIR} + ) +endif () \ No newline at end of file diff --git a/examples/gfind/gkfs_lib_example.cpp b/examples/user_library/gkfs_lib_example.cpp similarity index 100% rename from examples/gfind/gkfs_lib_example.cpp rename to examples/user_library/gkfs_lib_example.cpp -- GitLab From 6adbd340cbff6ae92a0591b35b2227368e6606db Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Sat, 8 Jun 2024 11:33:33 +0200 Subject: [PATCH 11/14] CMake: Always build examples --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 962878eee..16f2bc3ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -306,6 +306,7 @@ include_directories( add_subdirectory(src) add_subdirectory(include) +add_subdirectory(examples) ### Mark any CMake variables imported from {fmt} and spdlog as advanced, so ### that they don't appear in cmake-gui or ccmake. Similarly for FETCHCONTENT @@ -346,7 +347,6 @@ if (GKFS_BUILD_TESTS) message(STATUS "[gekkofs] Guided distributor tests: ${GKFS_TESTS_GUIDED_DISTRIBUTION}") add_subdirectory(tests) - add_subdirectory(examples) else() unset(GKFS_TESTS_INTERFACE CACHE) endif() -- GitLab From 5320d81dad0bccc521fc2fa3b64f76258043223b Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Sat, 8 Jun 2024 11:33:59 +0200 Subject: [PATCH 12/14] Fix user library headers --- examples/user_library/gkfs_lib_example.cpp | 26 ++++++++-------------- include/client/user_functions.hpp | 12 ++++++---- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/examples/user_library/gkfs_lib_example.cpp b/examples/user_library/gkfs_lib_example.cpp index 4644d29fc..3d422d2ae 100644 --- a/examples/user_library/gkfs_lib_example.cpp +++ b/examples/user_library/gkfs_lib_example.cpp @@ -1,6 +1,6 @@ /* - Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain - Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany This software was partially supported by the EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). @@ -25,23 +25,17 @@ SPDX-License-Identifier: GPL-3.0-or-later */ - +#include #include #include -#include #include -#include -#include -#include #include -#include -#include + +extern "C" { #include -#include -#include -#include -#include +#include +} using namespace std; @@ -53,18 +47,16 @@ write_file(std::string filename) { cout << "FD open " << fd << endl; char* bufwrite = (char*) malloc(10); - strncpy(bufwrite, "testing", 7); - + strncpy(bufwrite, "testing", 8); int size = gkfs::syscall::gkfs_write(fd, bufwrite, 7); - cout << "Writting size " << size << endl; + cout << "Written size " << size << endl; free(bufwrite); gkfs::syscall::gkfs_close(fd); } - void read_file(std::string filename) { int fdread = gkfs::syscall::gkfs_open(filename, S_IRWXU, O_RDONLY); diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp index 5d1bfdfe4..2fb5dd7f2 100644 --- a/include/client/user_functions.hpp +++ b/include/client/user_functions.hpp @@ -1,6 +1,6 @@ /* - Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain - Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany This software was partially supported by the EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). @@ -29,10 +29,14 @@ #ifndef GEKKOFS_USER_FUNCTIONS_HPP #define GEKKOFS_USER_FUNCTIONS_HPP -#include -#include #include #include +#include + +extern "C" { +#include +#include +} struct linux_dirent64; -- GitLab From 5aa4e850414fbcd0e57d0500eeafd63464a100e9 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Sat, 8 Jun 2024 11:35:16 +0200 Subject: [PATCH 13/14] Instruct the linker to mark the stack segment of gkfs_user_lib as non-executable. --- src/client/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index a4827af37..db59b16f4 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -73,6 +73,7 @@ target_sources( ) target_compile_definitions(gkfs_user_lib PUBLIC BYPASS_SYSCALL) +target_link_options(gkfs_user_lib PRIVATE -z noexecstack) if(GKFS_ENABLE_AGIOS) target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) -- GitLab From 92c904a266bcd2b27a5ba0cef74894a615f09e8b Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Sat, 8 Jun 2024 12:04:53 +0200 Subject: [PATCH 14/14] Added changelog --- CHANGELOG.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93e0446d5..2b5517121 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,15 +6,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### New +- Adding user library `gkfs_user_lib` that can be used to directly link to an application ([!171](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/171)). +- FMT10 and date removal, several dependencies updated. ([!172](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/172)). +- Fused GekkoFWD and GekkoFS. GekkoFWD is enabled with the `--enable-following` in the server configuration and the ENV variable + `LIBGKFS_FORWARDING_MAP_FILE` in the clients. ([!170](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/170)). - Replication without using the server. NUM_REPL (0 < NUM_REPL < num_servers) env variable defines the number of replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)). -- Modified write and reads to use a bitset instead of the traditional hash per chunk in the server. -- Added reattemp support in get_fs_config to other servers, when the initial server fails. -- Fused GekkoFWD and GekkoFS. GekkoFWD is enabled with the `--enable-following` in the server configuration and the ENV variable -`LIBGKFS_FORWARDING_MAP_FILE` in the clients. ([!170](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/170)). -- FMT10 and date removal, several dependencies updated. ([!172](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/172)). + - Modified write and reads to use a bitset instead of the traditional hash per chunk in the server. + - Added reattemp support in get_fs_config to other servers, when the initial server fails. + -### New ### Changed ### Removed ### Fixed -- GitLab