Code Maintenance: Configurations, definitions, adafs to gkfs

Restructuring code w.r.t. configurations and definitions:
- #defines have been mostly removed from configurations
- a dedicated config file has been added for configurations with constexpr
- past configure file is now only a cmake wrapper
- wrapping global functions into namespaces

Removed all adafs and ifs occurrences. Now called gkfs

CHANGELOG.md

@@ -86,7 +86,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Intercept I/O syscalls instead of GlibC function using [syscall intercept library](https://github.com/pmem/syscall_intercept)
 
 ## [0.4.0] - 2019-04-18
-First GekkoFS pubblic release
+First GekkoFS public release
 
 This version provides a client library that uses GLibC I/O function interception.
 

CMakeLists.txt

@@ -114,7 +114,7 @@ option(USE_SHM "Use shared memory for intra-node communication" OFF)
 message(STATUS "[gekkofs] Shared-memory communication: ${USE_SHM}")
 
 option(CREATE_CHECK_PARENTS "Check parent directory existance before creating child node" ON)
-message(STATUS "Create checks parents: ${CREATE_CHECK_PARENTS}")
+message(STATUS "[gekkofs] Create checks parents: ${CREATE_CHECK_PARENTS}")
 
 option(SYMLINK_SUPPORT "Compile with support for symlinks" ON)
 if(SYMLINK_SUPPORT)
@@ -146,7 +146,7 @@ add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${CLIENT_LOG_MESSAGE_SIZE})
 message(STATUS "[gekkofs] Maximum log message size in the client library: ${CLIENT_LOG_MESSAGE_SIZE}")
 mark_as_advanced(CLIENT_LOG_MESSAGE_SIZE)
 
-configure_file(include/global/configure.hpp.in include/global/configure.hpp)
+configure_file(include/global/cmake_configure.hpp.in include/global/cmake_configure.hpp)
 
 # Imported target
 add_library(RocksDB INTERFACE IMPORTED GLOBAL)

include/client/adafs_functions.hpp

-/*
-  Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain
-  Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany
-
-  This software was partially supported by the
-  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
-
-  This software was partially supported by the
-  ADA-FS project under the SPPEXA project funded by the DFG.
-
-  SPDX-License-Identifier: MIT
-*/
-
-#ifndef IFS_ADAFS_FUNCTIONS_HPP
-#define IFS_ADAFS_FUNCTIONS_HPP
-
-#include <client/open_file_map.hpp>
-#include <global/metadata.hpp>
-
-std::shared_ptr<Metadata> adafs_metadata(const std::string& path, bool follow_links = false);
-
-int adafs_open(const std::string& path, mode_t mode, int flags);
-
-int check_parent_dir(const std::string& path);
-
-int adafs_mk_node(const std::string& path, mode_t mode);
-
-int check_parent_dir(const std::string& path);
-
-int adafs_rm_node(const std::string& path);
-
-int adafs_access(const std::string& path, int mask, bool follow_links = true);
-
-int adafs_stat(const std::string& path, struct stat* buf, bool follow_links = true);
-
-int adafs_statvfs(struct statvfs* buf);
-
-int adafs_statfs(struct statfs* buf);
-
-off64_t adafs_lseek(unsigned int fd, off64_t offset, unsigned int whence);
-
-off64_t adafs_lseek(std::shared_ptr<OpenFile> adafs_fd, off64_t offset, unsigned int whence);
-
-int adafs_truncate(const std::string& path, off_t offset);
-
-int adafs_truncate(const std::string& path, off_t old_size, off_t new_size);
-
-int adafs_dup(int oldfd);
-
-int adafs_dup2(int oldfd, int newfd);
-
-#ifdef HAS_SYMLINKS
-int adafs_mk_symlink(const std::string& path, const std::string& target_path);
-int adafs_readlink(const std::string& path, char *buf, int bufsize);
-#endif
-
-
-ssize_t adafs_pwrite(std::shared_ptr<OpenFile> file,
-                     const char * buf, size_t count, off64_t offset);
-ssize_t adafs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset);
-ssize_t adafs_write(int fd, const void * buf, size_t count);
-ssize_t adafs_pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset);
-ssize_t adafs_writev(int fd, const struct iovec * iov, int iovcnt);
-
-ssize_t adafs_pread(std::shared_ptr<OpenFile> file, char * buf, size_t count, off64_t offset);
-ssize_t adafs_pread_ws(int fd, void* buf, size_t count, off64_t offset);
-ssize_t adafs_read(int fd, void* buf, size_t count);
-
-
-int adafs_opendir(const std::string& path);
-
-int getdents(unsigned int fd,
-             struct linux_dirent *dirp,
-             unsigned int count);
-
-int getdents64(unsigned int fd,
-             struct linux_dirent64 *dirp,
-             unsigned int count);
-
-int adafs_rmdir(const std::string& path);
-
-#endif //IFS_ADAFS_FUNCTIONS_HPP

include/client/env.hpp

@@ -14,7 +14,7 @@
 #ifndef GKFS_CLIENT_ENV
 #define GKFS_CLIENT_ENV
 
-#include <global/configure.hpp>
+#include <config.hpp>
 
 #define ADD_PREFIX(str) CLIENT_ENV_PREFIX str
 

include/client/gkfs_functions.hpp

+/*
+  Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain
+  Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany
+
+  This software was partially supported by the
+  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+  This software was partially supported by the
+  ADA-FS project under the SPPEXA project funded by the DFG.
+
+  SPDX-License-Identifier: MIT
+*/
+
+#ifndef GEKKOFS_GKFS_FUNCTIONS_HPP
+#define GEKKOFS_GKFS_FUNCTIONS_HPP
+
+#include <client/open_file_map.hpp>
+#include <global/metadata.hpp>
+
+std::shared_ptr<Metadata> gkfs_metadata(const std::string& path, bool follow_links = false);
+
+int check_parent_dir(const std::string& path);
+
+int gkfs_open(const std::string& path, mode_t mode, int flags);
+
+int gkfs_mk_node(const std::string& path, mode_t mode);
+
+int gkfs_rm_node(const std::string& path);
+
+int gkfs_access(const std::string& path, int mask, bool follow_links = true);
+
+int gkfs_stat(const std::string& path, struct stat* buf, bool follow_links = true);
+
+int gkfs_statvfs(struct statvfs* buf);
+
+int gkfs_statfs(struct statfs* buf);
+
+off64_t gkfs_lseek(unsigned int fd, off64_t offset, unsigned int whence);
+
+off64_t gkfs_lseek(std::shared_ptr<OpenFile> gkfs_fd, off64_t offset, unsigned int whence);
+
+int gkfs_truncate(const std::string& path, off_t offset);
+
+int gkfs_truncate(const std::string& path, off_t old_size, off_t new_size);
+
+int gkfs_dup(int oldfd);
+
+int gkfs_dup2(int oldfd, int newfd);
+
+#ifdef HAS_SYMLINKS
+
+int gkfs_mk_symlink(const std::string& path, const std::string& target_path);
+
+int gkfs_readlink(const std::string& path, char* buf, int bufsize);
+
+#endif
+
+
+ssize_t gkfs_pwrite(std::shared_ptr<OpenFile> file,
+                    const char* buf, size_t count, off64_t offset);
+
+ssize_t gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset);
+
+ssize_t gkfs_write(int fd, const void* buf, size_t count);
+
+ssize_t gkfs_pwritev(int fd, const struct iovec* iov, int iovcnt, off_t offset);
+
+ssize_t gkfs_writev(int fd, const struct iovec* iov, int iovcnt);
+
+ssize_t gkfs_pread(std::shared_ptr<OpenFile> file, char* buf, size_t count, off64_t offset);
+
+ssize_t gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset);
+
+ssize_t gkfs_read(int fd, void* buf, size_t count);
+
+
+int gkfs_opendir(const std::string& path);
+
+int getdents(unsigned int fd,
+             struct linux_dirent* dirp,
+             unsigned int count);
+
+int getdents64(unsigned int fd,
+               struct linux_dirent64* dirp,
+               unsigned int count);
+
+int gkfs_rmdir(const std::string& path);
+
+#endif //GEKKOFS_GKFS_FUNCTIONS_HPP

include/client/hooks.hpp

@@ -11,8 +11,8 @@
   SPDX-License-Identifier: MIT
 */
 
-#ifndef IFS_HOOKS_HPP
-#define IFS_HOOKS_HPP
+#ifndef GEKKOFS_HOOKS_HPP
+#define GEKKOFS_HOOKS_HPP
 
 #include <sys/types.h>
 #include <fcntl.h>
@@ -20,10 +20,14 @@
 
 int hook_openat(int dirfd, const char *cpath, int flags, mode_t mode);
 int hook_close(int fd);
+
 int hook_stat(const char* path, struct stat* buf);
+
 int hook_lstat(const char* path, struct stat* buf);
+
 int hook_fstat(unsigned int fd, struct stat* buf);
-int hook_fstatat(int dirfd, const char * cpath, struct stat * buf, int flags);
+
+int hook_fstatat(int dirfd, const char* cpath, struct stat* buf, int flags);
 int hook_read(unsigned int fd, void* buf, size_t count);
 int hook_pread(unsigned int fd, char * buf, size_t count, loff_t pos);
 int hook_write(unsigned int fd, const char * buf, size_t count);

include/client/intcp_functions.hpp

@@ -11,8 +11,8 @@
   SPDX-License-Identifier: MIT
 */
 
-#ifndef IFS_INTCP_FUNCTIONS_HPP
-#define IFS_INTCP_FUNCTIONS_HPP
+#ifndef GEKKOFS_INTCP_FUNCTIONS_HPP
+#define GEKKOFS_INTCP_FUNCTIONS_HPP
 
 #include <dirent.h>
 
@@ -75,6 +75,6 @@ strong_alias(intcp_statvfs, statvfs)
 int intcp_fstatvfs(int fd, struct statvfs *buf) noexcept;
 strong_alias(intcp_fstatvfs, fstatvfs)
 
-#endif // IFS_INTCP_FUNCTIONS_HPP
+#endif // GEKKOFS_INTCP_FUNCTIONS_HPP
 
 } // extern C

include/client/intercept.hpp

@@ -11,14 +11,14 @@
   SPDX-License-Identifier: MIT
 */
 
-#ifndef IFS_INTERCEPT_HPP
-#define IFS_INTERCEPT_HPP
+#ifndef GEKKOFS_INTERCEPT_HPP
+#define GEKKOFS_INTERCEPT_HPP
 
 int
 internal_hook_guard_wrapper(long syscall_number,
                             long arg0, long arg1, long arg2,
                             long arg3, long arg4, long arg5,
-                            long *syscall_return_value);
+                            long* syscall_return_value);
 
 int
 hook_guard_wrapper(long syscall_number,

include/client/open_dir.hpp

@@ -12,8 +12,8 @@
 */
 
 
-#ifndef IFS_OPEN_DIR_HPP
-#define IFS_OPEN_DIR_HPP
+#ifndef GEKKOFS_OPEN_DIR_HPP
+#define GEKKOFS_OPEN_DIR_HPP
 
 #include <string>
 #include <vector>
@@ -46,4 +46,4 @@ class OpenDir: public OpenFile {
 };
 
 
-#endif //IFS_OPEN_DIR_HPP
+#endif //GEKKOFS_OPEN_DIR_HPP

include/client/open_file_map.hpp

@@ -12,8 +12,8 @@
 */
 
 
-#ifndef IFS_OPEN_FILE_MAP_HPP
-#define IFS_OPEN_FILE_MAP_HPP
+#ifndef GEKKOFS_OPEN_FILE_MAP_HPP
+#define GEKKOFS_OPEN_FILE_MAP_HPP
 
 #include <map>
 #include <mutex>
@@ -115,4 +115,4 @@ public:
 };
 
 
-#endif //IFS_OPEN_FILE_MAP_HPP
+#endif //GEKKOFS_OPEN_FILE_MAP_HPP

include/client/preload_context.hpp

@@ -11,8 +11,8 @@
   SPDX-License-Identifier: MIT
 */
 
-#ifndef IFS_PRELOAD_CTX_HPP
-#define IFS_PRELOAD_CTX_HPP
+#ifndef GEKKOFS_PRELOAD_CTX_HPP
+#define GEKKOFS_PRELOAD_CTX_HPP
 
 #include <hermes.hpp>
 #include <map>
@@ -130,5 +130,5 @@ class PreloadContext {
 };
 
 
-#endif //IFS_PRELOAD_CTX_HPP
+#endif //GEKKOFS_PRELOAD_CTX_HPP
 

include/client/preload_util.hpp

@@ -12,15 +12,16 @@
 */
 
 
-#ifndef IFS_PRELOAD_UTIL_HPP
-#define IFS_PRELOAD_UTIL_HPP
+#ifndef GEKKOFS_PRELOAD_UTIL_HPP
+#define GEKKOFS_PRELOAD_UTIL_HPP
 
 #include <client/preload.hpp>
 #include <global/metadata.hpp>
-// third party libs
+
 #include <string>
 #include <iostream>
 #include <map>
+#include <type_traits>
 
 struct MetadentryUpdateFlags {
     bool atime = false;
@@ -60,14 +61,19 @@ extern hg_id_t rpc_mk_symlink_id;
 #endif
 
 // function definitions
+namespace gkfs {
+    namespace client {
+        template<typename E>
+        constexpr typename std::underlying_type<E>::type to_underlying(E e) {
+            return static_cast<typename std::underlying_type<E>::type>(e);
+        }
 
-int metadata_to_stat(const std::string& path, const Metadata& md, struct stat& attr);
-
-std::vector<std::pair<std::string, std::string>> load_hosts_file(const std::string& lfpath);
+        int metadata_to_stat(const std::string& path, const Metadata& md, struct stat& attr);
 
-hg_addr_t get_local_addr();
+        std::vector<std::pair<std::string, std::string>> load_hostfile(const std::string& lfpath);
 
-void load_hosts();
-bool lookup_all_hosts();
+        void load_hosts();
+    }
+}
 
-#endif //IFS_PRELOAD_UTIL_HPP
+#endif //GEKKOFS_PRELOAD_UTIL_HPP

include/client/rpc/hg_rpcs.hpp

@@ -77,7 +77,7 @@ struct fs_config {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::fs_config;
+    constexpr static const auto name = gkfs::hg_tag::fs_config;
 
     // requires response?
     constexpr static const auto requires_response = true;
@@ -256,7 +256,7 @@ struct create {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::create;
+    constexpr static const auto name = gkfs::hg_tag::create;
 
     // requires response?
     constexpr static const auto requires_response = true;
@@ -367,7 +367,7 @@ struct stat {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::stat;
+    constexpr static const auto name = gkfs::hg_tag::stat;
 
     // requires response?
     constexpr static const auto requires_response = true;
@@ -481,7 +481,7 @@ struct remove {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::remove;
+    constexpr static const auto name = gkfs::hg_tag::remove;
 
     // requires response?
     constexpr static const auto requires_response = true;
@@ -583,7 +583,7 @@ struct decr_size {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::decr_size;
+    constexpr static const auto name = gkfs::hg_tag::decr_size;
 
     // requires response?
     constexpr static const auto requires_response = true;
@@ -693,7 +693,7 @@ struct update_metadentry {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::update_metadentry;
+    constexpr static const auto name = gkfs::hg_tag::update_metadentry;
 
     // requires response?
     constexpr static const auto requires_response = true;
@@ -955,7 +955,7 @@ struct get_metadentry_size {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::get_metadentry_size;
+    constexpr static const auto name = gkfs::hg_tag::get_metadentry_size;
 
     // requires response?
     constexpr static const auto requires_response = true;
@@ -1066,7 +1066,7 @@ struct update_metadentry_size {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::update_metadentry_size;
+    constexpr static const auto name = gkfs::hg_tag::update_metadentry_size;
 
     // requires response?
     constexpr static const auto requires_response = true;
@@ -1206,7 +1206,7 @@ struct mk_symlink {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::mk_symlink;
+    constexpr static const auto name = gkfs::hg_tag::mk_symlink;
 
     // requires response?
     constexpr static const auto requires_response = true;
@@ -1319,7 +1319,7 @@ struct write_data {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::write_data;
+    constexpr static const auto name = gkfs::hg_tag::write_data;
 
     // requires response?
     constexpr static const auto requires_response = true;
@@ -1512,7 +1512,7 @@ struct read_data {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::read_data;
+    constexpr static const auto name = gkfs::hg_tag::read_data;
 
     // requires response?
     constexpr static const auto requires_response = true;
@@ -1705,7 +1705,7 @@ struct trunc_data {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::trunc_data;
+    constexpr static const auto name = gkfs::hg_tag::trunc_data;
 
     // requires response?
     constexpr static const auto requires_response = true;
@@ -1819,7 +1819,7 @@ struct get_dirents {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::get_dirents;
+    constexpr static const auto name = gkfs::hg_tag::get_dirents;
 
     // requires response?
     constexpr static const auto requires_response = true;
@@ -1942,7 +1942,7 @@ struct chunk_stat {
     constexpr static const hg_id_t mercury_id = public_id;
 
     // RPC name
-    constexpr static const auto name = hg_tag::chunk_stat;
+    constexpr static const auto name = gkfs::hg_tag::chunk_stat;
 
     // requires response?
     constexpr static const auto requires_response = true;

include/client/rpc/ld_rpc_data_ws.hpp

@@ -12,18 +12,19 @@
 */
 
 
-#ifndef IFS_PRELOAD_C_DATA_WS_HPP
-#define IFS_PRELOAD_C_DATA_WS_HPP
+#ifndef GEKKOFS_PRELOAD_C_DATA_WS_HPP
+#define GEKKOFS_PRELOAD_C_DATA_WS_HPP
 
 
 namespace rpc_send {
 
-ssize_t write(const std::string& path, const void* buf, const bool append_flag, const off64_t in_offset,
-                       const size_t write_size, const int64_t updated_metadentry_size);
-struct ChunkStat {
-    unsigned long chunk_size;
-    unsigned long chunk_total;
-    unsigned long chunk_free;
+    ssize_t write(const std::string& path, const void* buf, const bool append_flag, const off64_t in_offset,
+                  const size_t write_size, const int64_t updated_metadentry_size);
+
+    struct ChunkStat {
+        unsigned long chunk_size;
+        unsigned long chunk_total;
+        unsigned long chunk_free;
 };
 
 ssize_t read(const std::string& path, void* buf, const off64_t offset, const size_t read_size);
@@ -35,4 +36,4 @@ ChunkStat chunk_stat();
 }
 
 
-#endif //IFS_PRELOAD_C_DATA_WS_HPP
+#endif //GEKKOFS_PRELOAD_C_DATA_WS_HPP

include/client/rpc/ld_rpc_management.hpp

@@ -12,15 +12,15 @@
 */
 
 
-#ifndef IFS_MARGO_RPC_MANAGMENT_HPP
-#define IFS_MARGO_RPC_MANAGMENT_HPP
+#ifndef GEKKOFS_MARGO_RPC_MANAGMENT_HPP
+#define GEKKOFS_MARGO_RPC_MANAGMENT_HPP
 
 namespace rpc_send {
 
 
-bool get_fs_config();
+    bool get_fs_config();
 
 
 } // end namespace rpc_send
 
-#endif //IFS_MARGO_RPC_NANAGMENT_HPP
+#endif //GEKKOFS_MARGO_RPC_NANAGMENT_HPP

include/client/rpc/ld_rpc_metadentry.hpp

@@ -12,14 +12,16 @@
 */
 
 
-#ifndef IFS_PRELOAD_C_METADENTRY_HPP
-#define IFS_PRELOAD_C_METADENTRY_HPP
+#ifndef GEKKOFS_PRELOAD_C_METADENTRY_HPP
+#define GEKKOFS_PRELOAD_C_METADENTRY_HPP
 
 #include <string>
 
 /* Forward declaration */
 struct MetadentryUpdateFlags;
+
 class OpenDir;
+
 class Metadata;
 
 namespace rpc_send {
@@ -49,4 +51,4 @@ int mk_symlink(const std::string& path, const std::string& target_path);
 
 } // end namespace rpc_send
 
-#endif //IFS_PRELOAD_C_METADENTRY_HPP
+#endif //GEKKOFS_PRELOAD_C_METADENTRY_HPP

include/config.hpp

+/*
+  Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain
+  Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany
+
+  This software was partially supported by the
+  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+  This software was partially supported by the
+  ADA-FS project under the SPPEXA project funded by the DFG.
+
+  SPDX-License-Identifier: MIT
+*/
+
+#ifndef GEKKOFS_CONFIG_HPP
+#define GEKKOFS_CONFIG_HPP
+
+#include <global/cmake_configure.hpp>
+
+// environment prefixes (are concatenated in env module at compile time)
+#define CLIENT_ENV_PREFIX "LIBGKFS_"
+#define DAEMON_ENV_PREFIX "GKFS_"
+
+namespace gkfs_config {
+
+    constexpr auto hostfile_path = "./gkfs_hosts.txt";
+
+    namespace io {
+        /*
+         * Zero buffer before read. This is relevant if sparse files are used.
+         * If buffer is not zeroed, sparse regions contain invalid data.
+         */
+        constexpr auto zero_buffer_before_read = false;
+    }