Verified Commit 053cd9a5 authored by Marc Vef's avatar Marc Vef
Browse files

Code Maintenance: Configurations, definitions, adafs to gkfs

Restructuring code w.r.t. configurations and definitions:
- #defines have been mostly removed from configurations
- a dedicated config file has been added for configurations with constexpr
- past configure file is now only a cmake wrapper
- wrapping global functions into namespaces

Removed all adafs and ifs occurrences. Now called gkfs
parent 1ec0d0fd
Pipeline #810 failed with stages
in 7 minutes and 55 seconds
......@@ -86,7 +86,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Intercept I/O syscalls instead of GlibC function using [syscall intercept library](https://github.com/pmem/syscall_intercept)
## [0.4.0] - 2019-04-18
First GekkoFS pubblic release
First GekkoFS public release
This version provides a client library that uses GLibC I/O function interception.
......
......@@ -114,7 +114,7 @@ option(USE_SHM "Use shared memory for intra-node communication" OFF)
message(STATUS "[gekkofs] Shared-memory communication: ${USE_SHM}")
option(CREATE_CHECK_PARENTS "Check parent directory existance before creating child node" ON)
message(STATUS "Create checks parents: ${CREATE_CHECK_PARENTS}")
message(STATUS "[gekkofs] Create checks parents: ${CREATE_CHECK_PARENTS}")
option(SYMLINK_SUPPORT "Compile with support for symlinks" ON)
if(SYMLINK_SUPPORT)
......@@ -146,7 +146,7 @@ add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${CLIENT_LOG_MESSAGE_SIZE})
message(STATUS "[gekkofs] Maximum log message size in the client library: ${CLIENT_LOG_MESSAGE_SIZE}")
mark_as_advanced(CLIENT_LOG_MESSAGE_SIZE)
configure_file(include/global/configure.hpp.in include/global/configure.hpp)
configure_file(include/global/cmake_configure.hpp.in include/global/cmake_configure.hpp)
# Imported target
add_library(RocksDB INTERFACE IMPORTED GLOBAL)
......
/*
Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain
Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany
This software was partially supported by the
EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
This software was partially supported by the
ADA-FS project under the SPPEXA project funded by the DFG.
SPDX-License-Identifier: MIT
*/
#ifndef IFS_ADAFS_FUNCTIONS_HPP
#define IFS_ADAFS_FUNCTIONS_HPP
#include <client/open_file_map.hpp>
#include <global/metadata.hpp>
std::shared_ptr<Metadata> adafs_metadata(const std::string& path, bool follow_links = false);
int adafs_open(const std::string& path, mode_t mode, int flags);
int check_parent_dir(const std::string& path);
int adafs_mk_node(const std::string& path, mode_t mode);
int check_parent_dir(const std::string& path);
int adafs_rm_node(const std::string& path);
int adafs_access(const std::string& path, int mask, bool follow_links = true);
int adafs_stat(const std::string& path, struct stat* buf, bool follow_links = true);
int adafs_statvfs(struct statvfs* buf);
int adafs_statfs(struct statfs* buf);
off64_t adafs_lseek(unsigned int fd, off64_t offset, unsigned int whence);
off64_t adafs_lseek(std::shared_ptr<OpenFile> adafs_fd, off64_t offset, unsigned int whence);
int adafs_truncate(const std::string& path, off_t offset);
int adafs_truncate(const std::string& path, off_t old_size, off_t new_size);
int adafs_dup(int oldfd);
int adafs_dup2(int oldfd, int newfd);
#ifdef HAS_SYMLINKS
int adafs_mk_symlink(const std::string& path, const std::string& target_path);
int adafs_readlink(const std::string& path, char *buf, int bufsize);
#endif
ssize_t adafs_pwrite(std::shared_ptr<OpenFile> file,
const char * buf, size_t count, off64_t offset);
ssize_t adafs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset);
ssize_t adafs_write(int fd, const void * buf, size_t count);
ssize_t adafs_pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset);
ssize_t adafs_writev(int fd, const struct iovec * iov, int iovcnt);
ssize_t adafs_pread(std::shared_ptr<OpenFile> file, char * buf, size_t count, off64_t offset);
ssize_t adafs_pread_ws(int fd, void* buf, size_t count, off64_t offset);
ssize_t adafs_read(int fd, void* buf, size_t count);
int adafs_opendir(const std::string& path);
int getdents(unsigned int fd,
struct linux_dirent *dirp,
unsigned int count);
int getdents64(unsigned int fd,
struct linux_dirent64 *dirp,
unsigned int count);
int adafs_rmdir(const std::string& path);
#endif //IFS_ADAFS_FUNCTIONS_HPP
......@@ -14,7 +14,7 @@
#ifndef GKFS_CLIENT_ENV
#define GKFS_CLIENT_ENV
#include <global/configure.hpp>
#include <config.hpp>
#define ADD_PREFIX(str) CLIENT_ENV_PREFIX str
......
/*
Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain
Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany
This software was partially supported by the
EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
This software was partially supported by the
ADA-FS project under the SPPEXA project funded by the DFG.
SPDX-License-Identifier: MIT
*/
#ifndef GEKKOFS_GKFS_FUNCTIONS_HPP
#define GEKKOFS_GKFS_FUNCTIONS_HPP
#include <client/open_file_map.hpp>
#include <global/metadata.hpp>
std::shared_ptr<Metadata> gkfs_metadata(const std::string& path, bool follow_links = false);
int check_parent_dir(const std::string& path);
int gkfs_open(const std::string& path, mode_t mode, int flags);
int gkfs_mk_node(const std::string& path, mode_t mode);
int gkfs_rm_node(const std::string& path);
int gkfs_access(const std::string& path, int mask, bool follow_links = true);
int gkfs_stat(const std::string& path, struct stat* buf, bool follow_links = true);
int gkfs_statvfs(struct statvfs* buf);
int gkfs_statfs(struct statfs* buf);
off64_t gkfs_lseek(unsigned int fd, off64_t offset, unsigned int whence);
off64_t gkfs_lseek(std::shared_ptr<OpenFile> gkfs_fd, off64_t offset, unsigned int whence);
int gkfs_truncate(const std::string& path, off_t offset);
int gkfs_truncate(const std::string& path, off_t old_size, off_t new_size);
int gkfs_dup(int oldfd);
int gkfs_dup2(int oldfd, int newfd);
#ifdef HAS_SYMLINKS
int gkfs_mk_symlink(const std::string& path, const std::string& target_path);
int gkfs_readlink(const std::string& path, char* buf, int bufsize);
#endif
ssize_t gkfs_pwrite(std::shared_ptr<OpenFile> file,
const char* buf, size_t count, off64_t offset);
ssize_t gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset);
ssize_t gkfs_write(int fd, const void* buf, size_t count);
ssize_t gkfs_pwritev(int fd, const struct iovec* iov, int iovcnt, off_t offset);
ssize_t gkfs_writev(int fd, const struct iovec* iov, int iovcnt);
ssize_t gkfs_pread(std::shared_ptr<OpenFile> file, char* buf, size_t count, off64_t offset);
ssize_t gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset);
ssize_t gkfs_read(int fd, void* buf, size_t count);
int gkfs_opendir(const std::string& path);
int getdents(unsigned int fd,
struct linux_dirent* dirp,
unsigned int count);
int getdents64(unsigned int fd,
struct linux_dirent64* dirp,
unsigned int count);
int gkfs_rmdir(const std::string& path);
#endif //GEKKOFS_GKFS_FUNCTIONS_HPP
......@@ -11,8 +11,8 @@
SPDX-License-Identifier: MIT
*/
#ifndef IFS_HOOKS_HPP
#define IFS_HOOKS_HPP
#ifndef GEKKOFS_HOOKS_HPP
#define GEKKOFS_HOOKS_HPP
#include <sys/types.h>
#include <fcntl.h>
......@@ -20,10 +20,14 @@
int hook_openat(int dirfd, const char *cpath, int flags, mode_t mode);
int hook_close(int fd);
int hook_stat(const char* path, struct stat* buf);
int hook_lstat(const char* path, struct stat* buf);
int hook_fstat(unsigned int fd, struct stat* buf);
int hook_fstatat(int dirfd, const char * cpath, struct stat * buf, int flags);
int hook_fstatat(int dirfd, const char* cpath, struct stat* buf, int flags);
int hook_read(unsigned int fd, void* buf, size_t count);
int hook_pread(unsigned int fd, char * buf, size_t count, loff_t pos);
int hook_write(unsigned int fd, const char * buf, size_t count);
......
......@@ -11,8 +11,8 @@
SPDX-License-Identifier: MIT
*/
#ifndef IFS_INTCP_FUNCTIONS_HPP
#define IFS_INTCP_FUNCTIONS_HPP
#ifndef GEKKOFS_INTCP_FUNCTIONS_HPP
#define GEKKOFS_INTCP_FUNCTIONS_HPP
#include <dirent.h>
......@@ -75,6 +75,6 @@ strong_alias(intcp_statvfs, statvfs)
int intcp_fstatvfs(int fd, struct statvfs *buf) noexcept;
strong_alias(intcp_fstatvfs, fstatvfs)
#endif // IFS_INTCP_FUNCTIONS_HPP
#endif // GEKKOFS_INTCP_FUNCTIONS_HPP
} // extern C
......@@ -11,14 +11,14 @@
SPDX-License-Identifier: MIT
*/
#ifndef IFS_INTERCEPT_HPP
#define IFS_INTERCEPT_HPP
#ifndef GEKKOFS_INTERCEPT_HPP
#define GEKKOFS_INTERCEPT_HPP
int
internal_hook_guard_wrapper(long syscall_number,
long arg0, long arg1, long arg2,
long arg3, long arg4, long arg5,
long *syscall_return_value);
long* syscall_return_value);
int
hook_guard_wrapper(long syscall_number,
......
......@@ -12,8 +12,8 @@
*/
#ifndef IFS_OPEN_DIR_HPP
#define IFS_OPEN_DIR_HPP
#ifndef GEKKOFS_OPEN_DIR_HPP
#define GEKKOFS_OPEN_DIR_HPP
#include <string>
#include <vector>
......@@ -46,4 +46,4 @@ class OpenDir: public OpenFile {
};
#endif //IFS_OPEN_DIR_HPP
#endif //GEKKOFS_OPEN_DIR_HPP
......@@ -12,8 +12,8 @@
*/
#ifndef IFS_OPEN_FILE_MAP_HPP
#define IFS_OPEN_FILE_MAP_HPP
#ifndef GEKKOFS_OPEN_FILE_MAP_HPP
#define GEKKOFS_OPEN_FILE_MAP_HPP
#include <map>
#include <mutex>
......@@ -115,4 +115,4 @@ public:
};
#endif //IFS_OPEN_FILE_MAP_HPP
#endif //GEKKOFS_OPEN_FILE_MAP_HPP
......@@ -11,8 +11,8 @@
SPDX-License-Identifier: MIT
*/
#ifndef IFS_PRELOAD_CTX_HPP
#define IFS_PRELOAD_CTX_HPP
#ifndef GEKKOFS_PRELOAD_CTX_HPP
#define GEKKOFS_PRELOAD_CTX_HPP
#include <hermes.hpp>
#include <map>
......@@ -130,5 +130,5 @@ class PreloadContext {
};
#endif //IFS_PRELOAD_CTX_HPP
#endif //GEKKOFS_PRELOAD_CTX_HPP
......@@ -12,15 +12,16 @@
*/
#ifndef IFS_PRELOAD_UTIL_HPP
#define IFS_PRELOAD_UTIL_HPP
#ifndef GEKKOFS_PRELOAD_UTIL_HPP
#define GEKKOFS_PRELOAD_UTIL_HPP
#include <client/preload.hpp>
#include <global/metadata.hpp>
// third party libs
#include <string>
#include <iostream>
#include <map>
#include <type_traits>
struct MetadentryUpdateFlags {
bool atime = false;
......@@ -60,14 +61,19 @@ extern hg_id_t rpc_mk_symlink_id;
#endif
// function definitions
namespace gkfs {
namespace client {
template<typename E>
constexpr typename std::underlying_type<E>::type to_underlying(E e) {
return static_cast<typename std::underlying_type<E>::type>(e);
}
int metadata_to_stat(const std::string& path, const Metadata& md, struct stat& attr);
std::vector<std::pair<std::string, std::string>> load_hosts_file(const std::string& lfpath);
int metadata_to_stat(const std::string& path, const Metadata& md, struct stat& attr);
hg_addr_t get_local_addr();
std::vector<std::pair<std::string, std::string>> load_hostfile(const std::string& lfpath);
void load_hosts();
bool lookup_all_hosts();
void load_hosts();
}
}
#endif //IFS_PRELOAD_UTIL_HPP
#endif //GEKKOFS_PRELOAD_UTIL_HPP
......@@ -77,7 +77,7 @@ struct fs_config {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::fs_config;
constexpr static const auto name = gkfs::hg_tag::fs_config;
// requires response?
constexpr static const auto requires_response = true;
......@@ -256,7 +256,7 @@ struct create {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::create;
constexpr static const auto name = gkfs::hg_tag::create;
// requires response?
constexpr static const auto requires_response = true;
......@@ -367,7 +367,7 @@ struct stat {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::stat;
constexpr static const auto name = gkfs::hg_tag::stat;
// requires response?
constexpr static const auto requires_response = true;
......@@ -481,7 +481,7 @@ struct remove {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::remove;
constexpr static const auto name = gkfs::hg_tag::remove;
// requires response?
constexpr static const auto requires_response = true;
......@@ -583,7 +583,7 @@ struct decr_size {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::decr_size;
constexpr static const auto name = gkfs::hg_tag::decr_size;
// requires response?
constexpr static const auto requires_response = true;
......@@ -693,7 +693,7 @@ struct update_metadentry {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::update_metadentry;
constexpr static const auto name = gkfs::hg_tag::update_metadentry;
// requires response?
constexpr static const auto requires_response = true;
......@@ -955,7 +955,7 @@ struct get_metadentry_size {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::get_metadentry_size;
constexpr static const auto name = gkfs::hg_tag::get_metadentry_size;
// requires response?
constexpr static const auto requires_response = true;
......@@ -1066,7 +1066,7 @@ struct update_metadentry_size {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::update_metadentry_size;
constexpr static const auto name = gkfs::hg_tag::update_metadentry_size;
// requires response?
constexpr static const auto requires_response = true;
......@@ -1206,7 +1206,7 @@ struct mk_symlink {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::mk_symlink;
constexpr static const auto name = gkfs::hg_tag::mk_symlink;
// requires response?
constexpr static const auto requires_response = true;
......@@ -1319,7 +1319,7 @@ struct write_data {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::write_data;
constexpr static const auto name = gkfs::hg_tag::write_data;
// requires response?
constexpr static const auto requires_response = true;
......@@ -1512,7 +1512,7 @@ struct read_data {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::read_data;
constexpr static const auto name = gkfs::hg_tag::read_data;
// requires response?
constexpr static const auto requires_response = true;
......@@ -1705,7 +1705,7 @@ struct trunc_data {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::trunc_data;
constexpr static const auto name = gkfs::hg_tag::trunc_data;
// requires response?
constexpr static const auto requires_response = true;
......@@ -1819,7 +1819,7 @@ struct get_dirents {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::get_dirents;
constexpr static const auto name = gkfs::hg_tag::get_dirents;
// requires response?
constexpr static const auto requires_response = true;
......@@ -1942,7 +1942,7 @@ struct chunk_stat {
constexpr static const hg_id_t mercury_id = public_id;
// RPC name
constexpr static const auto name = hg_tag::chunk_stat;
constexpr static const auto name = gkfs::hg_tag::chunk_stat;
// requires response?
constexpr static const auto requires_response = true;
......
......@@ -12,18 +12,19 @@
*/
#ifndef IFS_PRELOAD_C_DATA_WS_HPP
#define IFS_PRELOAD_C_DATA_WS_HPP
#ifndef GEKKOFS_PRELOAD_C_DATA_WS_HPP
#define GEKKOFS_PRELOAD_C_DATA_WS_HPP
namespace rpc_send {
ssize_t write(const std::string& path, const void* buf, const bool append_flag, const off64_t in_offset,
const size_t write_size, const int64_t updated_metadentry_size);
struct ChunkStat {
unsigned long chunk_size;
unsigned long chunk_total;
unsigned long chunk_free;
ssize_t write(const std::string& path, const void* buf, const bool append_flag, const off64_t in_offset,
const size_t write_size, const int64_t updated_metadentry_size);
struct ChunkStat {
unsigned long chunk_size;
unsigned long chunk_total;
unsigned long chunk_free;
};
ssize_t read(const std::string& path, void* buf, const off64_t offset, const size_t read_size);
......@@ -35,4 +36,4 @@ ChunkStat chunk_stat();
}
#endif //IFS_PRELOAD_C_DATA_WS_HPP
#endif //GEKKOFS_PRELOAD_C_DATA_WS_HPP
......@@ -12,15 +12,15 @@
*/
#ifndef IFS_MARGO_RPC_MANAGMENT_HPP
#define IFS_MARGO_RPC_MANAGMENT_HPP
#ifndef GEKKOFS_MARGO_RPC_MANAGMENT_HPP
#define GEKKOFS_MARGO_RPC_MANAGMENT_HPP
namespace rpc_send {
bool get_fs_config();
bool get_fs_config();
} // end namespace rpc_send
#endif //IFS_MARGO_RPC_NANAGMENT_HPP
#endif //GEKKOFS_MARGO_RPC_NANAGMENT_HPP
......@@ -12,14 +12,16 @@
*/
#ifndef IFS_PRELOAD_C_METADENTRY_HPP
#define IFS_PRELOAD_C_METADENTRY_HPP
#ifndef GEKKOFS_PRELOAD_C_METADENTRY_HPP
#define GEKKOFS_PRELOAD_C_METADENTRY_HPP
#include <string>
/* Forward declaration */
struct MetadentryUpdateFlags;
class OpenDir;
class Metadata;
namespace rpc_send {
......@@ -49,4 +51,4 @@ int mk_symlink(const std::string& path, const std::string& target_path);
} // end namespace rpc_send
#endif //IFS_PRELOAD_C_METADENTRY_HPP
#endif //GEKKOFS_PRELOAD_C_METADENTRY_HPP
/*
Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain
Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany
This software was partially supported by the
EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
This software was partially supported by the
ADA-FS project under the SPPEXA project funded by the DFG.
SPDX-License-Identifier: MIT
*/
#ifndef GEKKOFS_CONFIG_HPP
#define GEKKOFS_CONFIG_HPP
#include <global/cmake_configure.hpp>
// environment prefixes (are concatenated in env module at compile time)
#define CLIENT_ENV_PREFIX "LIBGKFS_"
#define DAEMON_ENV_PREFIX "GKFS_"
namespace gkfs_config {
constexpr auto hostfile_path = "./gkfs_hosts.txt";
namespace io {
/*
* Zero buffer before read. This is relevant if sparse files are used.
* If buffer is not zeroed, sparse regions contain invalid data.
*/
constexpr auto zero_buffer_before_read = false;
}