From 22eda4c1e0a265411db00de0548ef4fa856e8ca8 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 27 Jun 2024 16:57:42 +0200 Subject: [PATCH 1/7] Client readdir cache: Base structure --- include/client/CMakeLists.txt | 2 + include/client/cache.hpp | 65 ++++++++++++++++++++++++++++++ include/client/preload_context.hpp | 19 +++++++++ src/client/CMakeLists.txt | 2 + src/client/cache.cpp | 65 ++++++++++++++++++++++++++++++ src/client/preload.cpp | 11 +++++ src/client/preload_context.cpp | 20 +++++++++ 7 files changed, 184 insertions(+) create mode 100644 include/client/cache.hpp create mode 100644 src/client/cache.cpp diff --git a/include/client/CMakeLists.txt b/include/client/CMakeLists.txt index e4ce8983c..7e419713f 100644 --- a/include/client/CMakeLists.txt +++ b/include/client/CMakeLists.txt @@ -42,6 +42,7 @@ target_sources( preload.hpp preload_context.hpp preload_util.hpp + cache.hpp rpc/rpc_types.hpp rpc/forward_management.hpp rpc/forward_metadata.hpp @@ -68,6 +69,7 @@ target_sources( preload.hpp preload_context.hpp preload_util.hpp + cache.hpp rpc/rpc_types.hpp rpc/forward_management.hpp rpc/forward_metadata.hpp diff --git a/include/client/cache.hpp b/include/client/cache.hpp new file mode 100644 index 000000000..aa5197a4c --- /dev/null +++ b/include/client/cache.hpp @@ -0,0 +1,65 @@ +/* + Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS' POSIX interface. + + GekkoFS' POSIX interface is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GekkoFS' POSIX interface is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with GekkoFS' POSIX interface. If not, see + . + + SPDX-License-Identifier: LGPL-3.0-or-later +*/ + +#ifndef GKFS_CLIENT_CACHE +#define GKFS_CLIENT_CACHE + +#include +#include +#include +#include + +namespace gkfs::cache { + +class Cache { +private: + std::unordered_map entries_; + std::mutex mtx_; + +public: + Cache() = default; + + virtual ~Cache() = default; + + void + insert(const std::string& key, const std::string& value); + + std::optional + get(const std::string& key); + + void + remove(const std::string& key); + + void + clear(); +}; + +} // namespace gkfs::cache + +#endif // GKFS_CLIENT_CACHE diff --git a/include/client/preload_context.hpp b/include/client/preload_context.hpp index 95d2f8093..1a8b9aabd 100644 --- a/include/client/preload_context.hpp +++ b/include/client/preload_context.hpp @@ -55,6 +55,10 @@ namespace messagepack { class ClientMetrics; } +namespace cache { +class Cache; +} + namespace preload { /* * Client file system config @@ -90,6 +94,8 @@ private: std::shared_ptr ofm_; std::shared_ptr distributor_; std::shared_ptr fs_conf_; + std::shared_ptr cache_; + bool use_cache_{false}; std::string cwd_; std::vector mountdir_components_; @@ -229,6 +235,19 @@ public: const std::shared_ptr& fs_conf() const; + std::shared_ptr + cache() const; + + void + cache(std::shared_ptr cache); + + bool + use_cache() const; + + void + use_cache(bool use_cache); + + void enable_interception(); diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index e2a53a082..2622fd57a 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -47,6 +47,7 @@ target_sources(gkfs_intercept preload.cpp preload_context.cpp preload_util.cpp + cache.cpp rpc/rpc_types.cpp rpc/forward_data.cpp rpc/forward_data_proxy.cpp @@ -68,6 +69,7 @@ target_sources( preload_context.cpp preload_util.cpp malleability.cpp + cache.cpp rpc/rpc_types.cpp rpc/forward_data.cpp rpc/forward_data_proxy.cpp diff --git a/src/client/cache.cpp b/src/client/cache.cpp new file mode 100644 index 000000000..feaf7c638 --- /dev/null +++ b/src/client/cache.cpp @@ -0,0 +1,65 @@ +/* + Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS' POSIX interface. + + GekkoFS' POSIX interface is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. + + GekkoFS' POSIX interface is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with GekkoFS' POSIX interface. If not, see + . + + SPDX-License-Identifier: LGPL-3.0-or-later +*/ + +#include +#include +#include +#include + +namespace gkfs::cache { + +void +Cache::insert(const std::string& key, const std::string& value) { + std::lock_guard const lock(mtx_); + entries_[key] = value; +} + +std::optional +Cache::get(const std::string& key) { + std::lock_guard const lock(mtx_); + // return key if found + if(entries_.find(key) != entries_.end()) { + return entries_[key]; + } + return {}; +} + +void +Cache::remove(const std::string& key) { + std::lock_guard const lock(mtx_); + entries_.erase(key); +} + +void +Cache::clear() { + std::lock_guard const lock(mtx_); + entries_.clear(); +} + +} // namespace gkfs::cache diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 32a52ad56..54e089094 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -257,6 +258,16 @@ init_environment() { #endif CTX->distributor(distributor); } + try { + LOG(INFO, "Initializing client caching..."); + auto cache = std::make_shared(); + CTX->cache(cache); + LOG(INFO, "Client caching is enabled and will be used!"); + CTX->use_cache(true); + } catch(const std::exception& e) { + exit_error_msg(EXIT_FAILURE, + "Failed to initialize cache: "s + e.what()); + } LOG(INFO, "Retrieving file system configuration..."); diff --git a/src/client/preload_context.cpp b/src/client/preload_context.cpp index 7ae774a90..cf6979d79 100644 --- a/src/client/preload_context.cpp +++ b/src/client/preload_context.cpp @@ -412,6 +412,26 @@ PreloadContext::fs_conf() const { return fs_conf_; } +std::shared_ptr +PreloadContext::cache() const { + return cache_; +} + +void +PreloadContext::cache(std::shared_ptr cache) { + cache_ = cache; +} + +bool +PreloadContext::use_cache() const { + return use_cache_; +} + +void +PreloadContext::use_cache(bool use_cache) { + use_cache_ = use_cache; +} + void PreloadContext::enable_interception() { interception_enabled_ = true; -- GitLab From b2b74091f03fe30474b101ac54cd9c21b28e22c3 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Sun, 28 Apr 2024 17:20:20 +0200 Subject: [PATCH 2/7] Adding directory entry cache avoiding stat RPCs. Flushed for dir on dir close. This uses the extended dir RPC call. --- include/client/cache.hpp | 35 +++++++++++++--- include/client/env.hpp | 1 + src/client/cache.cpp | 77 ++++++++++++++++++++++++++++++----- src/client/gkfs_functions.cpp | 52 +++++++++++++++++++++-- src/client/preload.cpp | 20 +++++---- src/client/preload_util.cpp | 31 ++++++++++++++ 6 files changed, 188 insertions(+), 28 deletions(-) diff --git a/include/client/cache.hpp b/include/client/cache.hpp index aa5197a4c..8ebf6b335 100644 --- a/include/client/cache.hpp +++ b/include/client/cache.hpp @@ -30,17 +30,37 @@ #ifndef GKFS_CLIENT_CACHE #define GKFS_CLIENT_CACHE +#include + +#include +#include #include #include #include #include +#include namespace gkfs::cache { +struct cache_entry { + gkfs::filemap::FileType file_type; + uint64_t size; + time_t ctime; +}; + class Cache { private: - std::unordered_map entries_; + std::unordered_map> + entries_; + std::unordered_map entry_dir_id_; std::mutex mtx_; + std::hash str_hash; + + uint32_t + gen_dir_id(const std::string& dir_path); + + uint32_t + get_dir_id(const std::string& dir_path); public: Cache() = default; @@ -48,18 +68,23 @@ public: virtual ~Cache() = default; void - insert(const std::string& key, const std::string& value); + insert(const std::string& parent_dir, const std::string name, + const cache_entry value); + + std::optional + get(const std::string& parent_dir, const std::string& name); - std::optional - get(const std::string& key); + void + clear_dir(const std::string& dir_path); void - remove(const std::string& key); + dump_cache_to_log(const std::string& dir_path); void clear(); }; + } // namespace gkfs::cache #endif // GKFS_CLIENT_CACHE diff --git a/include/client/env.hpp b/include/client/env.hpp index 5574df247..33532ef61 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -60,6 +60,7 @@ static constexpr auto METRICS_IP_PORT = ADD_PREFIX("METRICS_IP_PORT"); static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL"); static constexpr auto PROXY_PID_FILE = ADD_PREFIX("PROXY_PID_FILE"); +static constexpr auto DIR_CACHE = ADD_PREFIX("ENABLE_DIR_CACHE"); } // namespace gkfs::env diff --git a/src/client/cache.cpp b/src/client/cache.cpp index feaf7c638..23da22991 100644 --- a/src/client/cache.cpp +++ b/src/client/cache.cpp @@ -28,38 +28,95 @@ */ #include +#include +#include +#include + +#include #include #include #include +#include namespace gkfs::cache { +uint32_t +Cache::gen_dir_id(const std::string& dir_path) { + return str_hash(dir_path); +} + +uint32_t +Cache::get_dir_id(const std::string& dir_path) { + // check if id already exists in map and return + if(entry_dir_id_.find(dir_path) != entry_dir_id_.end()) { + return entry_dir_id_[dir_path]; + } + // otherwise generate one + auto dir_id = gen_dir_id(dir_path); + entry_dir_id_.emplace(dir_path, dir_id); + return dir_id; +} + + void -Cache::insert(const std::string& key, const std::string& value) { +Cache::insert(const std::string& parent_dir, const std::string name, + const cache_entry value) { + std::lock_guard const lock(mtx_); + auto dir_id = get_dir_id(parent_dir); + entries_[dir_id].emplace(name, value); +} + +std::optional +Cache::get(const std::string& parent_dir, const std::string& name) { std::lock_guard const lock(mtx_); - entries_[key] = value; + auto dir_id = get_dir_id(parent_dir); + if(entries_[dir_id].find(name) != entries_[dir_id].end()) { + return entries_[dir_id][name]; + } else { + return {}; + } } -std::optional -Cache::get(const std::string& key) { +void +Cache::clear_dir(const std::string& dir_path) { std::lock_guard const lock(mtx_); - // return key if found - if(entries_.find(key) != entries_.end()) { - return entries_[key]; + + auto id_it = entry_dir_id_.find(dir_path); + if(id_it == entry_dir_id_.end()) { + return; + } + auto entry_it = entries_.find(id_it->second); + if(entry_it != entries_.end()) { + entries_.erase(entry_it); } - return {}; + entry_dir_id_.erase(id_it); } void -Cache::remove(const std::string& key) { +Cache::dump_cache_to_log(const std::string& dir_path) { std::lock_guard const lock(mtx_); - entries_.erase(key); + auto id_it = entry_dir_id_.find(dir_path); + if(id_it == entry_dir_id_.end()) { + LOG(INFO, "{}(): Cache contents for dir path '{}' NONE", __func__, + dir_path); + return; + } + auto dir_id = id_it->second; + for(auto& [name, entry] : entries_[dir_id]) { + // log entry + LOG(INFO, + "{}(): Cache contents for dir path '{}' -> name '{}' is_dir '{}' size '{}' ctime '{}'", + __func__, dir_path, name, + entry.file_type == gkfs::filemap::FileType::directory, entry.size, + entry.ctime); + } } void Cache::clear() { std::lock_guard const lock(mtx_); entries_.clear(); + entry_dir_id_.clear(); } } // namespace gkfs::cache diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index ec04145f1..81be54cd1 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #ifdef GKFS_ENABLE_CLIENT_METRICS @@ -1326,8 +1327,45 @@ gkfs_opendir(const std::string& path) { errno = ENOTDIR; return -1; } - - auto ret = gkfs::rpc::forward_get_dirents(path); + pair> ret{}; + // Use cache: Aka get all entries from all servers for the basic metadata + // this is used in get_metadata() later to avoid stat RPCs + if(CTX->use_cache()) { + ret.second = make_shared(path); + // TODO parallelize + for(uint64_t i = 0; i < CTX->hosts().size(); i++) { + pair, bool, + size_t, time_t>>>> + res{}; + if(gkfs::config::proxy::fwd_get_dirents_single && + CTX->use_proxy()) { + res = gkfs::rpc::forward_get_dirents_single_proxy(path, i); + } else { + res = gkfs::rpc::forward_get_dirents_single(path, i); + } + // auto res = gkfs::rpc::forward_get_dirents_single(path, + // i); + auto& open_dir = *res.second; + for(auto& dentry : open_dir) { + // type returns as a boolean. true if it is a directory + LOG(DEBUG, "name: {} type: {} size: {} ctime: {}", + get<0>(dentry), get<1>(dentry), get<2>(dentry), + get<3>(dentry)); + auto ftype = get<1>(dentry) ? gkfs::filemap::FileType::directory + : gkfs::filemap::FileType::regular; + // filename, is_dir, size, ctime + ret.second->add(get<0>(dentry), ftype); + CTX->cache()->insert(path, get<0>(dentry), + gkfs::cache::cache_entry{ftype, + get<2>(dentry), + get<3>(dentry)}); + } + ret.first = res.first; + } + // CTX->cache()->dump_cache_to_log(path); + } else { + ret = gkfs::rpc::forward_get_dirents(path); + } auto err = ret.first; if(err) { errno = err; @@ -1389,7 +1427,6 @@ gkfs_rmdir(const std::string& path) { */ int gkfs_getdents(unsigned int fd, struct linux_dirent* dirp, unsigned int count) { - // Get opendir object (content was downloaded with opendir() call) auto open_dir = CTX->file_map()->get_dir(fd); if(open_dir == nullptr) { @@ -1464,7 +1501,6 @@ gkfs_getdents(unsigned int fd, struct linux_dirent* dirp, unsigned int count) { int gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, unsigned int count) { - auto open_dir = CTX->file_map()->get_dir(fd); if(open_dir == nullptr) { // Cast did not succeeded: open_file is a regular file @@ -1533,6 +1569,14 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, int gkfs_close(unsigned int fd) { if(CTX->file_map()->exist(fd)) { + if(CTX->use_cache()) { + // clear cache for directory + if(CTX->file_map()->get(fd)->type() == + gkfs::filemap::FileType::directory) { + CTX->cache()->clear_dir(CTX->file_map()->get(fd)->path()); + } + // CTX->cache()->dump_cache_to_log(CTX->file_map()->get(fd)->path()); + } // No call to the daemon is required CTX->file_map()->remove(fd); return 0; diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 54e089094..b112c2906 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -258,15 +258,17 @@ init_environment() { #endif CTX->distributor(distributor); } - try { - LOG(INFO, "Initializing client caching..."); - auto cache = std::make_shared(); - CTX->cache(cache); - LOG(INFO, "Client caching is enabled and will be used!"); - CTX->use_cache(true); - } catch(const std::exception& e) { - exit_error_msg(EXIT_FAILURE, - "Failed to initialize cache: "s + e.what()); + if(gkfs::env::var_is_set(gkfs::env::DIR_CACHE)) { + try { + LOG(INFO, "Initializing client caching..."); + auto cache = std::make_shared(); + CTX->cache(cache); + LOG(INFO, "Client caching is enabled and will be used!"); + CTX->use_cache(true); + } catch(const std::exception& e) { + exit_error_msg(EXIT_FAILURE, + "Failed to initialize cache: "s + e.what()); + } } LOG(INFO, "Retrieving file system configuration..."); diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index 806db6639..b0210abdf 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -45,6 +46,7 @@ #include #include #include +#include extern "C" { #include @@ -209,6 +211,35 @@ optional get_metadata(const string& path, bool follow_links) { std::string attr; int err{}; + if(CTX->use_cache()) { + std::filesystem::path p(path); + auto parent = p.parent_path().string(); + auto filename = p.filename().string(); + // LOG(INFO, "{}(): for path '{}' -> parent path '{}' leaf name + // '{}'", + // __func__, path, p.parent_path().string(), + // p.filename().string()); + auto cache_entry = CTX->cache()->get(parent, filename); + if(cache_entry) { + // LOG(INFO, "{}(): Cache hit for path '{}'", __func__, + // path); + // TOOD something like this: + // struct stat st{}; + // metadata_to_stat(path, *cache_entry, st); + // return gkfs::metadata::Metadata{st}; + // TODO add mode to extended RPC + mode_t mode = 33188; + if(cache_entry->file_type == gkfs::filemap::FileType::directory) { + mode = 16895; + } + gkfs::metadata::Metadata md{}; + md.mode(mode); + md.ctime(cache_entry->ctime); + md.size(cache_entry->size); + return md; + } + } + if(gkfs::config::proxy::fwd_stat && CTX->use_proxy()) { err = gkfs::rpc::forward_stat_proxy(path, attr); } else { -- GitLab From f37b3e7b639e48e99be9990311e71eb576dc1d96 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Mon, 29 Apr 2024 19:32:33 +0200 Subject: [PATCH 3/7] Parallelize opendir when using dir cache --- src/client/gkfs_functions.cpp | 63 ++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 81be54cd1..3ad8c72b3 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1332,19 +1332,25 @@ gkfs_opendir(const std::string& path) { // this is used in get_metadata() later to avoid stat RPCs if(CTX->use_cache()) { ret.second = make_shared(path); - // TODO parallelize + + std::vector, + bool, size_t, time_t>>>>>> + futures; + // Launch RPC calls asynchronously for(uint64_t i = 0; i < CTX->hosts().size(); i++) { - pair, bool, - size_t, time_t>>>> - res{}; - if(gkfs::config::proxy::fwd_get_dirents_single && - CTX->use_proxy()) { - res = gkfs::rpc::forward_get_dirents_single_proxy(path, i); - } else { - res = gkfs::rpc::forward_get_dirents_single(path, i); - } - // auto res = gkfs::rpc::forward_get_dirents_single(path, - // i); + futures.push_back(std::async(std::launch::async, [&, i]() { + if(gkfs::config::proxy::fwd_get_dirents_single && + CTX->use_proxy()) { + return gkfs::rpc::forward_get_dirents_single_proxy(path, i); + } else { + return gkfs::rpc::forward_get_dirents_single(path, i); + } + })); + } + // Collect and process results + for(auto& fut : futures) { + auto res = fut.get(); // Wait for the RPC result auto& open_dir = *res.second; for(auto& dentry : open_dir) { // type returns as a boolean. true if it is a directory @@ -1362,6 +1368,39 @@ gkfs_opendir(const std::string& path) { } ret.first = res.first; } + // TODO parallelize + // for(uint64_t i = 0; i < CTX->hosts().size(); i++) { + // pair, bool, + // size_t, time_t>>>> + // res{}; + // if(gkfs::config::proxy::fwd_get_dirents_single && + // CTX->use_proxy()) { + // res = + // gkfs::rpc::forward_get_dirents_single_proxy(path, i); + // } else { + // res = gkfs::rpc::forward_get_dirents_single(path, i); + // } + // auto& open_dir = *res.second; + // for(auto& dentry : open_dir) { + // // type returns as a boolean. true if it is a + // directory LOG(DEBUG, "name: {} type: {} size: {} + // ctime: {}", + // get<0>(dentry), get<1>(dentry), get<2>(dentry), + // get<3>(dentry)); + // auto ftype = get<1>(dentry) ? + // gkfs::filemap::FileType::directory + // : + // gkfs::filemap::FileType::regular; + // // filename, is_dir, size, ctime + // ret.second->add(get<0>(dentry), ftype); + // CTX->cache()->insert(path, get<0>(dentry), + // gkfs::cache::cache_entry{ftype, + // get<2>(dentry), + // get<3>(dentry)}); + // } + // ret.first = res.first; + // } // CTX->cache()->dump_cache_to_log(path); } else { ret = gkfs::rpc::forward_get_dirents(path); -- GitLab From 03426a3774da1798d826602d26211bef7b3af321 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 27 Jun 2024 16:58:33 +0200 Subject: [PATCH 4/7] Adding config for async opendir --- include/config.hpp | 1 + src/client/gkfs_functions.cpp | 123 +++++++++++++++++----------------- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/include/config.hpp b/include/config.hpp index e859b20f3..5c081ffd1 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -138,6 +138,7 @@ constexpr auto chunksize = 524288; // in bytes (e.g., 524288 == 512KB) // size of preallocated buffer to hold directory entries in rpc call constexpr auto dirents_buff_size = (8 * 1024 * 1024); // 8 mega constexpr auto dirents_buff_size_proxy = (128 * 1024 * 1024); // 8 mega +constexpr auto async_opendir = true; /* * Indicates the number of concurrent progress to drive I/O operations of chunk * files to and from local file systems The value is directly mapped to created diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 3ad8c72b3..06cfcf7a1 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1332,75 +1332,74 @@ gkfs_opendir(const std::string& path) { // this is used in get_metadata() later to avoid stat RPCs if(CTX->use_cache()) { ret.second = make_shared(path); - - std::vector, + bool, size_t, time_t>>>>>> + futures; + // Launch RPC calls asynchronously + for(uint64_t i = 0; i < CTX->hosts().size(); i++) { + futures.push_back(std::async(std::launch::async, [&, i]() { + if(gkfs::config::proxy::fwd_get_dirents_single && + CTX->use_proxy()) { + return gkfs::rpc::forward_get_dirents_single_proxy(path, + i); + } else { + return gkfs::rpc::forward_get_dirents_single(path, i); + } + })); + } + // Collect and process results + for(auto& fut : futures) { + auto res = fut.get(); // Wait for the RPC result + auto& open_dir = *res.second; + for(auto& dentry : open_dir) { + // type returns as a boolean. true if it is a directory + LOG(DEBUG, "name: {} type: {} size: {} ctime: {}", + get<0>(dentry), get<1>(dentry), get<2>(dentry), + get<3>(dentry)); + auto ftype = get<1>(dentry) + ? gkfs::filemap::FileType::directory + : gkfs::filemap::FileType::regular; + // filename, is_dir, size, ctime + ret.second->add(get<0>(dentry), ftype); + CTX->cache()->insert( + path, get<0>(dentry), + gkfs::cache::cache_entry{ftype, get<2>(dentry), + get<3>(dentry)}); + } + ret.first = res.first; + } + } else { + for(uint64_t i = 0; i < CTX->hosts().size(); i++) { pair, - bool, size_t, time_t>>>>>> - futures; - // Launch RPC calls asynchronously - for(uint64_t i = 0; i < CTX->hosts().size(); i++) { - futures.push_back(std::async(std::launch::async, [&, i]() { + bool, size_t, time_t>>>> + res{}; if(gkfs::config::proxy::fwd_get_dirents_single && CTX->use_proxy()) { - return gkfs::rpc::forward_get_dirents_single_proxy(path, i); + res = gkfs::rpc::forward_get_dirents_single_proxy(path, i); } else { - return gkfs::rpc::forward_get_dirents_single(path, i); + res = gkfs::rpc::forward_get_dirents_single(path, i); } - })); - } - // Collect and process results - for(auto& fut : futures) { - auto res = fut.get(); // Wait for the RPC result - auto& open_dir = *res.second; - for(auto& dentry : open_dir) { - // type returns as a boolean. true if it is a directory - LOG(DEBUG, "name: {} type: {} size: {} ctime: {}", - get<0>(dentry), get<1>(dentry), get<2>(dentry), - get<3>(dentry)); - auto ftype = get<1>(dentry) ? gkfs::filemap::FileType::directory - : gkfs::filemap::FileType::regular; - // filename, is_dir, size, ctime - ret.second->add(get<0>(dentry), ftype); - CTX->cache()->insert(path, get<0>(dentry), - gkfs::cache::cache_entry{ftype, - get<2>(dentry), - get<3>(dentry)}); + auto& open_dir = *res.second; + for(auto& dentry : open_dir) { + // LOG(DEBUG, "name: {} type: {} size: {} + // ctime {} ", + // get<0>(dentry), get<1>(dentry), + // get<2>(dentry), get<3>(dentry)); + auto ftype = get<1>(dentry) + ? gkfs::filemap::FileType::directory + : gkfs::filemap::FileType::regular; + // filename, is_dir, size, ctime + ret.second->add(get<0>(dentry), ftype); + CTX->cache()->insert( + path, get<0>(dentry), + gkfs::cache::cache_entry{ftype, get<2>(dentry), + get<3>(dentry)}); + } + ret.first = res.first; } - ret.first = res.first; } - // TODO parallelize - // for(uint64_t i = 0; i < CTX->hosts().size(); i++) { - // pair, bool, - // size_t, time_t>>>> - // res{}; - // if(gkfs::config::proxy::fwd_get_dirents_single && - // CTX->use_proxy()) { - // res = - // gkfs::rpc::forward_get_dirents_single_proxy(path, i); - // } else { - // res = gkfs::rpc::forward_get_dirents_single(path, i); - // } - // auto& open_dir = *res.second; - // for(auto& dentry : open_dir) { - // // type returns as a boolean. true if it is a - // directory LOG(DEBUG, "name: {} type: {} size: {} - // ctime: {}", - // get<0>(dentry), get<1>(dentry), get<2>(dentry), - // get<3>(dentry)); - // auto ftype = get<1>(dentry) ? - // gkfs::filemap::FileType::directory - // : - // gkfs::filemap::FileType::regular; - // // filename, is_dir, size, ctime - // ret.second->add(get<0>(dentry), ftype); - // CTX->cache()->insert(path, get<0>(dentry), - // gkfs::cache::cache_entry{ftype, - // get<2>(dentry), - // get<3>(dentry)}); - // } - // ret.first = res.first; - // } // CTX->cache()->dump_cache_to_log(path); } else { ret = gkfs::rpc::forward_get_dirents(path); -- GitLab From 22e94669f3748620ff807691b956a5afd3661d15 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Thu, 27 Jun 2024 17:05:14 +0200 Subject: [PATCH 5/7] Renamed Cache to DentryCache. Added several options to config.hpp Env variable overrides config.hpp --- include/client/cache.hpp | 14 +++++++++--- include/client/env.hpp | 2 +- include/client/preload_context.hpp | 21 +++++++++++------- include/config.hpp | 8 +++++++ src/client/cache.cpp | 20 +++++++++++------- src/client/gkfs_functions.cpp | 34 +++++++++++++++++++----------- src/client/preload.cpp | 23 ++++++++++++++------ src/client/preload_context.cpp | 19 +++++++++-------- src/client/preload_util.cpp | 4 ++-- 9 files changed, 95 insertions(+), 50 deletions(-) diff --git a/include/client/cache.hpp b/include/client/cache.hpp index 8ebf6b335..b9d1dfd76 100644 --- a/include/client/cache.hpp +++ b/include/client/cache.hpp @@ -42,16 +42,20 @@ namespace gkfs::cache { +namespace dir { + struct cache_entry { gkfs::filemap::FileType file_type; uint64_t size; time_t ctime; }; -class Cache { +class DentryCache { private: + // > std::unordered_map> entries_; + // std::unordered_map entry_dir_id_; std::mutex mtx_; std::hash str_hash; @@ -63,9 +67,9 @@ private: get_dir_id(const std::string& dir_path); public: - Cache() = default; + DentryCache() = default; - virtual ~Cache() = default; + virtual ~DentryCache() = default; void insert(const std::string& parent_dir, const std::string name, @@ -83,6 +87,10 @@ public: void clear(); }; +} // namespace dir + +//// > +// std::unordered_map> size_cache; } // namespace gkfs::cache diff --git a/include/client/env.hpp b/include/client/env.hpp index 33532ef61..a2ddd1fcd 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -60,7 +60,7 @@ static constexpr auto METRICS_IP_PORT = ADD_PREFIX("METRICS_IP_PORT"); static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL"); static constexpr auto PROXY_PID_FILE = ADD_PREFIX("PROXY_PID_FILE"); -static constexpr auto DIR_CACHE = ADD_PREFIX("ENABLE_DIR_CACHE"); +static constexpr auto DISABLE_DENTRY_CACHE = ADD_PREFIX("DISABLE_DENTRY_CACHE"); } // namespace gkfs::env diff --git a/include/client/preload_context.hpp b/include/client/preload_context.hpp index 1a8b9aabd..a2d7e15be 100644 --- a/include/client/preload_context.hpp +++ b/include/client/preload_context.hpp @@ -56,7 +56,12 @@ class ClientMetrics; } namespace cache { -class Cache; +namespace dir { +class DentryCache; +} +namespace file { +class WriteSizeCache; +} } namespace preload { @@ -94,8 +99,8 @@ private: std::shared_ptr ofm_; std::shared_ptr distributor_; std::shared_ptr fs_conf_; - std::shared_ptr cache_; - bool use_cache_{false}; + std::shared_ptr dentry_cache_; + bool use_dentry_cache_{false}; std::string cwd_; std::vector mountdir_components_; @@ -235,17 +240,17 @@ public: const std::shared_ptr& fs_conf() const; - std::shared_ptr - cache() const; + std::shared_ptr + dentry_cache() const; void - cache(std::shared_ptr cache); + dentry_cache(std::shared_ptr dentry_cache); bool - use_cache() const; + use_dentry_cache() const; void - use_cache(bool use_cache); + use_dentry_cache(bool use_dentry_cache); void diff --git a/include/config.hpp b/include/config.hpp index 5c081ffd1..10943ad7e 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -46,6 +46,14 @@ constexpr auto hostfile_path = "./gkfs_hosts.txt"; // We do not default this, ENV variable always required. constexpr auto forwarding_file_path = ""; +namespace cache { +// Optimization for readdir which avoids consecutive stat calls +constexpr bool use_dentry_cache = true; +// When enabled, the dentry cache is cleared when a directory is closed. +// Disabling this may cause semantic issues. +constexpr bool clear_dentry_cache_on_close = true; +} // namespace cache + namespace client_metrics { // Default directory where client metrics are stored. Can be set via // LIBGKFS_METRICS_PATH. Filename consists of starting time, pid, and hostname diff --git a/src/client/cache.cpp b/src/client/cache.cpp index 23da22991..c3335d05e 100644 --- a/src/client/cache.cpp +++ b/src/client/cache.cpp @@ -40,13 +40,15 @@ namespace gkfs::cache { +namespace dir { + uint32_t -Cache::gen_dir_id(const std::string& dir_path) { +DentryCache::gen_dir_id(const std::string& dir_path) { return str_hash(dir_path); } uint32_t -Cache::get_dir_id(const std::string& dir_path) { +DentryCache::get_dir_id(const std::string& dir_path) { // check if id already exists in map and return if(entry_dir_id_.find(dir_path) != entry_dir_id_.end()) { return entry_dir_id_[dir_path]; @@ -59,15 +61,15 @@ Cache::get_dir_id(const std::string& dir_path) { void -Cache::insert(const std::string& parent_dir, const std::string name, - const cache_entry value) { +DentryCache::insert(const std::string& parent_dir, const std::string name, + const cache_entry value) { std::lock_guard const lock(mtx_); auto dir_id = get_dir_id(parent_dir); entries_[dir_id].emplace(name, value); } std::optional -Cache::get(const std::string& parent_dir, const std::string& name) { +DentryCache::get(const std::string& parent_dir, const std::string& name) { std::lock_guard const lock(mtx_); auto dir_id = get_dir_id(parent_dir); if(entries_[dir_id].find(name) != entries_[dir_id].end()) { @@ -78,7 +80,7 @@ Cache::get(const std::string& parent_dir, const std::string& name) { } void -Cache::clear_dir(const std::string& dir_path) { +DentryCache::clear_dir(const std::string& dir_path) { std::lock_guard const lock(mtx_); auto id_it = entry_dir_id_.find(dir_path); @@ -93,7 +95,7 @@ Cache::clear_dir(const std::string& dir_path) { } void -Cache::dump_cache_to_log(const std::string& dir_path) { +DentryCache::dump_cache_to_log(const std::string& dir_path) { std::lock_guard const lock(mtx_); auto id_it = entry_dir_id_.find(dir_path); if(id_it == entry_dir_id_.end()) { @@ -113,10 +115,12 @@ Cache::dump_cache_to_log(const std::string& dir_path) { } void -Cache::clear() { +DentryCache::clear() { std::lock_guard const lock(mtx_); entries_.clear(); entry_dir_id_.clear(); } +} // namespace dir + } // namespace gkfs::cache diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 06cfcf7a1..f4b20a110 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1323,20 +1323,23 @@ gkfs_opendir(const std::string& path) { } if(!S_ISDIR(md->mode())) { - LOG(DEBUG, "Path is not a directory"); + LOG(DEBUG, "{}() Path is not a directory", __func__); errno = ENOTDIR; return -1; } pair> ret{}; // Use cache: Aka get all entries from all servers for the basic metadata // this is used in get_metadata() later to avoid stat RPCs - if(CTX->use_cache()) { + if(CTX->use_dentry_cache()) { ret.second = make_shared(path); if constexpr(gkfs::config::rpc::async_opendir) { std::vector, bool, size_t, time_t>>>>>> futures; + LOG(DEBUG, + "{}() Sending async dirents for path '{}' to '{}' daemons ...", + __func__, path, CTX->hosts().size()); // Launch RPC calls asynchronously for(uint64_t i = 0; i < CTX->hosts().size(); i++) { futures.push_back(std::async(std::launch::async, [&, i]() { @@ -1349,6 +1352,7 @@ gkfs_opendir(const std::string& path) { } })); } + int cnt = 0; // Collect and process results for(auto& fut : futures) { auto res = fut.get(); // Wait for the RPC result @@ -1363,13 +1367,17 @@ gkfs_opendir(const std::string& path) { : gkfs::filemap::FileType::regular; // filename, is_dir, size, ctime ret.second->add(get<0>(dentry), ftype); - CTX->cache()->insert( + CTX->dentry_cache()->insert( path, get<0>(dentry), - gkfs::cache::cache_entry{ftype, get<2>(dentry), - get<3>(dentry)}); + gkfs::cache::dir::cache_entry{ftype, get<2>(dentry), + get<3>(dentry)}); + cnt++; } ret.first = res.first; } + LOG(DEBUG, + "{}() Unpacked dirents for path '{}' counted '{}' entries", + __func__, path, cnt); } else { for(uint64_t i = 0; i < CTX->hosts().size(); i++) { pair, @@ -1392,15 +1400,15 @@ gkfs_opendir(const std::string& path) { : gkfs::filemap::FileType::regular; // filename, is_dir, size, ctime ret.second->add(get<0>(dentry), ftype); - CTX->cache()->insert( + CTX->dentry_cache()->insert( path, get<0>(dentry), - gkfs::cache::cache_entry{ftype, get<2>(dentry), - get<3>(dentry)}); + gkfs::cache::dir::cache_entry{ftype, get<2>(dentry), + get<3>(dentry)}); } ret.first = res.first; } } - // CTX->cache()->dump_cache_to_log(path); + // CTX->dentry_cache()->dump_cache_to_log(path); } else { ret = gkfs::rpc::forward_get_dirents(path); } @@ -1607,13 +1615,15 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, int gkfs_close(unsigned int fd) { if(CTX->file_map()->exist(fd)) { - if(CTX->use_cache()) { + if(CTX->use_dentry_cache() && + gkfs::config::cache::clear_dentry_cache_on_close) { // clear cache for directory if(CTX->file_map()->get(fd)->type() == gkfs::filemap::FileType::directory) { - CTX->cache()->clear_dir(CTX->file_map()->get(fd)->path()); + CTX->dentry_cache()->clear_dir( + CTX->file_map()->get(fd)->path()); } - // CTX->cache()->dump_cache_to_log(CTX->file_map()->get(fd)->path()); + // CTX->dentry_cache()->dump_cache_to_log(CTX->file_map()->get(fd)->path()); } // No call to the daemon is required CTX->file_map()->remove(fd); diff --git a/src/client/preload.cpp b/src/client/preload.cpp index b112c2906..4d44095af 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -258,16 +258,25 @@ init_environment() { #endif CTX->distributor(distributor); } - if(gkfs::env::var_is_set(gkfs::env::DIR_CACHE)) { + + if(gkfs::config::cache::use_dentry_cache && + !gkfs::env::var_is_set(gkfs::env::DISABLE_DENTRY_CACHE)) { try { - LOG(INFO, "Initializing client caching..."); - auto cache = std::make_shared(); - CTX->cache(cache); - LOG(INFO, "Client caching is enabled and will be used!"); - CTX->use_cache(true); + LOG(INFO, "Initializing dentry caching..."); + auto dentry_cache = + std::make_shared(); + CTX->dentry_cache(dentry_cache); + LOG(INFO, "dentry caching enabled."); + CTX->use_dentry_cache(true); } catch(const std::exception& e) { exit_error_msg(EXIT_FAILURE, - "Failed to initialize cache: "s + e.what()); + "Failed to initialize dentry cache: "s + e.what()); + } + } else { + if(gkfs::env::var_is_set(gkfs::env::DISABLE_DENTRY_CACHE)) { + LOG(INFO, "Dentry cache is disabled by environment variable."); + } else { + LOG(INFO, "Dentry cache is disabled by configuration."); } } diff --git a/src/client/preload_context.cpp b/src/client/preload_context.cpp index cf6979d79..ccc41a0ea 100644 --- a/src/client/preload_context.cpp +++ b/src/client/preload_context.cpp @@ -412,24 +412,25 @@ PreloadContext::fs_conf() const { return fs_conf_; } -std::shared_ptr -PreloadContext::cache() const { - return cache_; +std::shared_ptr +PreloadContext::dentry_cache() const { + return dentry_cache_; } void -PreloadContext::cache(std::shared_ptr cache) { - cache_ = cache; +PreloadContext::dentry_cache( + std::shared_ptr dentry_cache) { + dentry_cache_ = dentry_cache; } bool -PreloadContext::use_cache() const { - return use_cache_; +PreloadContext::use_dentry_cache() const { + return use_dentry_cache_; } void -PreloadContext::use_cache(bool use_cache) { - use_cache_ = use_cache; +PreloadContext::use_dentry_cache(bool use_dentry_cache) { + use_dentry_cache_ = use_dentry_cache; } void diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index b0210abdf..da1cad01e 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -211,7 +211,7 @@ optional get_metadata(const string& path, bool follow_links) { std::string attr; int err{}; - if(CTX->use_cache()) { + if(CTX->use_dentry_cache()) { std::filesystem::path p(path); auto parent = p.parent_path().string(); auto filename = p.filename().string(); @@ -219,7 +219,7 @@ get_metadata(const string& path, bool follow_links) { // '{}'", // __func__, path, p.parent_path().string(), // p.filename().string()); - auto cache_entry = CTX->cache()->get(parent, filename); + auto cache_entry = CTX->dentry_cache()->get(parent, filename); if(cache_entry) { // LOG(INFO, "{}(): Cache hit for path '{}'", __func__, // path); -- GitLab From a89f6ed36c3c6c365928d94a6f2020c37e6d0bb0 Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Mon, 15 Jul 2024 17:52:36 +0200 Subject: [PATCH 6/7] Cleanup, Readme, Changelog, Added env variable to LIBGKFS_DIR_CACHE --- CHANGELOG.md | 3 + README.md | 3 + include/client/CMakeLists.txt | 2 +- include/client/cache.hpp | 63 +++++++++++++---- include/client/env.hpp | 2 +- include/client/preload_context.hpp | 5 +- include/common/common_defs.hpp | 2 + include/config.hpp | 3 +- src/client/cache.cpp | 3 + src/client/gkfs_functions.cpp | 108 ++++++++++------------------- src/client/preload.cpp | 9 ++- src/client/preload_util.cpp | 20 ++---- 12 files changed, 114 insertions(+), 109 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40a68d266..cc028bc4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ### New +- Added a directory cache for the file system client to improve `ls -l` type operations by avoiding consecutive stat calls + ([!194](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/194)). + - The cache is experimental and thus disabled by default and can be enabled with the env variable `LIBGKFS_DISABLE_DIR_CACHE` set to `ON`. - Added file system expansion support ([!196](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/196)). - Added the tool `gkfs_malleability` to steer start, status, and finalize requests for expansion operations. - `-DGKFS_BUILD_TOOLS=ON` must be set for CMake to build the tool. diff --git a/README.md b/README.md index 0343fff79..6201fa964 100644 --- a/README.md +++ b/README.md @@ -517,6 +517,9 @@ Client-metrics require the CMake argument `-DGKFS_ENABLE_CLIENT_METRICS=ON` (see - `LIBGKFS_METRICS_IP_PORT` - Enable flushing to a set ZeroMQ server (replaces `LIBGKFS_METRICS_PATH`). - `LIBGKFS_PROXY_PID_FILE` - Path to the proxy pid file (when using the GekkoFS proxy). - `LIBGKFS_NUM_REPL` - Number of replicas for data. +#### Caching +- `LIBGKFS_DENTRY_CACHE` - Enable caching directory entries until closing the directory (default: OFF). +Improves performance for `ls -l` type operations. Further compile-time settings available at `include/config.hpp`. ### Daemon #### Logging diff --git a/include/client/CMakeLists.txt b/include/client/CMakeLists.txt index 7e419713f..d76a24a99 100644 --- a/include/client/CMakeLists.txt +++ b/include/client/CMakeLists.txt @@ -42,7 +42,7 @@ target_sources( preload.hpp preload_context.hpp preload_util.hpp - cache.hpp + cache.hpp rpc/rpc_types.hpp rpc/forward_management.hpp rpc/forward_metadata.hpp diff --git a/include/client/cache.hpp b/include/client/cache.hpp index b9d1dfd76..87a5de813 100644 --- a/include/client/cache.hpp +++ b/include/client/cache.hpp @@ -44,25 +44,44 @@ namespace gkfs::cache { namespace dir { +/** + * @brief Cache entry metadata. + * The entries are limited to the get_dir_extended RPC. + */ struct cache_entry { gkfs::filemap::FileType file_type; uint64_t size; time_t ctime; }; +/** + * @brief Cache for directory entries to accelerate ls -l type operations + */ class DentryCache { private: - // > + // >: Associate a directory id with its entries + // containing the directory name and cache entry metadata std::unordered_map> entries_; - // + // : Associate a directory path with a unique id std::unordered_map entry_dir_id_; - std::mutex mtx_; - std::hash str_hash; - + std::mutex mtx_; // Mutex to protect the cache + std::hash str_hash; // hash to generate ids + + /** + * @brief Generate a unique id for caching a directory + * @param dir_path + * @return id + */ uint32_t gen_dir_id(const std::string& dir_path); + /** + * @brief Get the unique id for a directory to retrieve its entries. Creates + * an id if it does not exist. + * @param dir_path + * @return id + */ uint32_t get_dir_id(const std::string& dir_path); @@ -71,28 +90,48 @@ public: virtual ~DentryCache() = default; + /** + * @brief Insert a new entry in the cache + * @param parent_dir + * @param name + * @param value + */ void - insert(const std::string& parent_dir, const std::string name, - const cache_entry value); - + insert(const std::string& parent_dir, std::string name, cache_entry value); + + /** + * @brief Get an entry from the cache for a given directory + * @param parent_dir + * @param name + * @return std::optional + */ std::optional get(const std::string& parent_dir, const std::string& name); + /** + * @brief Clear the cache for a given directory. Called when a directory is + * closed + * @param dir_path + */ void clear_dir(const std::string& dir_path); + /** + * @brief Dump the cache to the log for debugging purposes. Not used in + * production. + * @param dir_path + */ void dump_cache_to_log(const std::string& dir_path); + /** + * @brief Clear the entire cache + */ void clear(); }; } // namespace dir -//// > -// std::unordered_map> size_cache; - - } // namespace gkfs::cache #endif // GKFS_CLIENT_CACHE diff --git a/include/client/env.hpp b/include/client/env.hpp index a2ddd1fcd..231d9c6ad 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -60,7 +60,7 @@ static constexpr auto METRICS_IP_PORT = ADD_PREFIX("METRICS_IP_PORT"); static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL"); static constexpr auto PROXY_PID_FILE = ADD_PREFIX("PROXY_PID_FILE"); -static constexpr auto DISABLE_DENTRY_CACHE = ADD_PREFIX("DISABLE_DENTRY_CACHE"); +static constexpr auto DENTRY_CACHE = ADD_PREFIX("DENTRY_CACHE"); } // namespace gkfs::env diff --git a/include/client/preload_context.hpp b/include/client/preload_context.hpp index a2d7e15be..662b09244 100644 --- a/include/client/preload_context.hpp +++ b/include/client/preload_context.hpp @@ -59,10 +59,7 @@ namespace cache { namespace dir { class DentryCache; } -namespace file { -class WriteSizeCache; -} -} +} // namespace cache namespace preload { /* diff --git a/include/common/common_defs.hpp b/include/common/common_defs.hpp index 58a67f17c..bd1647801 100644 --- a/include/common/common_defs.hpp +++ b/include/common/common_defs.hpp @@ -120,6 +120,8 @@ constexpr auto migrate_metadata = "rpc_srv_migrate_metadata"; namespace config::syscall::stat { // Number 512-byte blocks allocated as it is in the linux kernel (struct_stat.h) constexpr auto st_nblocksize = 512; +constexpr auto file_mode_default = 33188; +constexpr auto dir_mode_default = 16895; } // namespace config::syscall::stat } // namespace gkfs #endif // GEKKOFS_COMMON_DEFS_HPP diff --git a/include/config.hpp b/include/config.hpp index 10943ad7e..899beff7b 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -48,7 +48,7 @@ constexpr auto forwarding_file_path = ""; namespace cache { // Optimization for readdir which avoids consecutive stat calls -constexpr bool use_dentry_cache = true; +constexpr bool use_dentry_cache = false; // When enabled, the dentry cache is cleared when a directory is closed. // Disabling this may cause semantic issues. constexpr bool clear_dentry_cache_on_close = true; @@ -146,7 +146,6 @@ constexpr auto chunksize = 524288; // in bytes (e.g., 524288 == 512KB) // size of preallocated buffer to hold directory entries in rpc call constexpr auto dirents_buff_size = (8 * 1024 * 1024); // 8 mega constexpr auto dirents_buff_size_proxy = (128 * 1024 * 1024); // 8 mega -constexpr auto async_opendir = true; /* * Indicates the number of concurrent progress to drive I/O operations of chunk * files to and from local file systems The value is directly mapped to created diff --git a/src/client/cache.cpp b/src/client/cache.cpp index c3335d05e..ad75d4c9a 100644 --- a/src/client/cache.cpp +++ b/src/client/cache.cpp @@ -44,6 +44,9 @@ namespace dir { uint32_t DentryCache::gen_dir_id(const std::string& dir_path) { + // While collisions can theoretically occur, they are extremely unlikely as + // clients are ephemeral and thus the lifetime of a cached directory as + // well. return str_hash(dir_path); } diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index f4b20a110..a5cc35100 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1328,87 +1328,52 @@ gkfs_opendir(const std::string& path) { return -1; } pair> ret{}; - // Use cache: Aka get all entries from all servers for the basic metadata + // Use cache: Get all entries from all servers for the basic metadata // this is used in get_metadata() later to avoid stat RPCs if(CTX->use_dentry_cache()) { ret.second = make_shared(path); - if constexpr(gkfs::config::rpc::async_opendir) { - std::vector, - bool, size_t, time_t>>>>>> - futures; - LOG(DEBUG, - "{}() Sending async dirents for path '{}' to '{}' daemons ...", - __func__, path, CTX->hosts().size()); - // Launch RPC calls asynchronously - for(uint64_t i = 0; i < CTX->hosts().size(); i++) { - futures.push_back(std::async(std::launch::async, [&, i]() { - if(gkfs::config::proxy::fwd_get_dirents_single && - CTX->use_proxy()) { - return gkfs::rpc::forward_get_dirents_single_proxy(path, - i); - } else { - return gkfs::rpc::forward_get_dirents_single(path, i); - } - })); - } - int cnt = 0; - // Collect and process results - for(auto& fut : futures) { - auto res = fut.get(); // Wait for the RPC result - auto& open_dir = *res.second; - for(auto& dentry : open_dir) { - // type returns as a boolean. true if it is a directory - LOG(DEBUG, "name: {} type: {} size: {} ctime: {}", - get<0>(dentry), get<1>(dentry), get<2>(dentry), - get<3>(dentry)); - auto ftype = get<1>(dentry) - ? gkfs::filemap::FileType::directory - : gkfs::filemap::FileType::regular; - // filename, is_dir, size, ctime - ret.second->add(get<0>(dentry), ftype); - CTX->dentry_cache()->insert( - path, get<0>(dentry), - gkfs::cache::dir::cache_entry{ftype, get<2>(dentry), - get<3>(dentry)}); - cnt++; - } - ret.first = res.first; - } - LOG(DEBUG, - "{}() Unpacked dirents for path '{}' counted '{}' entries", - __func__, path, cnt); - } else { - for(uint64_t i = 0; i < CTX->hosts().size(); i++) { + std::vector, - bool, size_t, time_t>>>> - res{}; + bool, size_t, time_t>>>>>> + dcache_futures; + LOG(DEBUG, + "{}() Sending async dirents for path '{}' to '{}' daemons ...", + __func__, path, CTX->hosts().size()); + // Launch RPC calls asynchronously + for(uint64_t i = 0; i < CTX->hosts().size(); i++) { + dcache_futures.push_back(std::async(std::launch::async, [&, i]() { if(gkfs::config::proxy::fwd_get_dirents_single && CTX->use_proxy()) { - res = gkfs::rpc::forward_get_dirents_single_proxy(path, i); + return gkfs::rpc::forward_get_dirents_single_proxy(path, i); } else { - res = gkfs::rpc::forward_get_dirents_single(path, i); - } - auto& open_dir = *res.second; - for(auto& dentry : open_dir) { - // LOG(DEBUG, "name: {} type: {} size: {} - // ctime {} ", - // get<0>(dentry), get<1>(dentry), - // get<2>(dentry), get<3>(dentry)); - auto ftype = get<1>(dentry) - ? gkfs::filemap::FileType::directory - : gkfs::filemap::FileType::regular; - // filename, is_dir, size, ctime - ret.second->add(get<0>(dentry), ftype); - CTX->dentry_cache()->insert( - path, get<0>(dentry), - gkfs::cache::dir::cache_entry{ftype, get<2>(dentry), - get<3>(dentry)}); + return gkfs::rpc::forward_get_dirents_single(path, i); } - ret.first = res.first; + })); + } + int cnt = 0; + // Collect and process results + for(auto& fut : dcache_futures) { + auto res = fut.get(); // Wait for the RPC result + auto& open_dir = *res.second; + for(auto& dentry : open_dir) { + // type returns as a boolean. true if it is a directory + LOG(DEBUG, "name: {} type: {} size: {} ctime: {}", + get<0>(dentry), get<1>(dentry), get<2>(dentry), + get<3>(dentry)); + auto ftype = get<1>(dentry) ? gkfs::filemap::FileType::directory + : gkfs::filemap::FileType::regular; + // filename, is_dir, size, ctime + ret.second->add(get<0>(dentry), ftype); + CTX->dentry_cache()->insert( + path, get<0>(dentry), + gkfs::cache::dir::cache_entry{ftype, get<2>(dentry), + get<3>(dentry)}); + cnt++; } + ret.first = res.first; } - // CTX->dentry_cache()->dump_cache_to_log(path); + LOG(DEBUG, "{}() Unpacked dirents for path '{}' counted '{}' entries", + __func__, path, cnt); } else { ret = gkfs::rpc::forward_get_dirents(path); } @@ -1623,7 +1588,6 @@ gkfs_close(unsigned int fd) { CTX->dentry_cache()->clear_dir( CTX->file_map()->get(fd)->path()); } - // CTX->dentry_cache()->dump_cache_to_log(CTX->file_map()->get(fd)->path()); } // No call to the daemon is required CTX->file_map()->remove(fd); diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 4d44095af..52422c057 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -259,8 +259,11 @@ init_environment() { CTX->distributor(distributor); } - if(gkfs::config::cache::use_dentry_cache && - !gkfs::env::var_is_set(gkfs::env::DISABLE_DENTRY_CACHE)) { + auto use_dcache = gkfs::env::get_var(gkfs::env::DENTRY_CACHE, + gkfs::config::cache::use_dentry_cache + ? "ON" + : "OFF") == "ON"; + if(use_dcache) { try { LOG(INFO, "Initializing dentry caching..."); auto dentry_cache = @@ -273,7 +276,7 @@ init_environment() { "Failed to initialize dentry cache: "s + e.what()); } } else { - if(gkfs::env::var_is_set(gkfs::env::DISABLE_DENTRY_CACHE)) { + if(gkfs::env::var_is_set(gkfs::env::DENTRY_CACHE)) { LOG(INFO, "Dentry cache is disabled by environment variable."); } else { LOG(INFO, "Dentry cache is disabled by configuration."); diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index da1cad01e..3737368f8 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -211,26 +211,19 @@ optional get_metadata(const string& path, bool follow_links) { std::string attr; int err{}; + // Use file metadata from dentry cache if available if(CTX->use_dentry_cache()) { + // get parent and filename path to retrieve the cache entry std::filesystem::path p(path); auto parent = p.parent_path().string(); auto filename = p.filename().string(); - // LOG(INFO, "{}(): for path '{}' -> parent path '{}' leaf name - // '{}'", - // __func__, path, p.parent_path().string(), - // p.filename().string()); auto cache_entry = CTX->dentry_cache()->get(parent, filename); if(cache_entry) { - // LOG(INFO, "{}(): Cache hit for path '{}'", __func__, - // path); - // TOOD something like this: - // struct stat st{}; - // metadata_to_stat(path, *cache_entry, st); - // return gkfs::metadata::Metadata{st}; - // TODO add mode to extended RPC - mode_t mode = 33188; + LOG(DEBUG, "{}(): Dentry cache hit for file '{}'", __func__, path); + // if cache_entry exists, generate a Metadata object from it. + mode_t mode = gkfs::config::syscall::stat::file_mode_default; if(cache_entry->file_type == gkfs::filemap::FileType::directory) { - mode = 16895; + mode = gkfs::config::syscall::stat::dir_mode_default; } gkfs::metadata::Metadata md{}; md.mode(mode); @@ -239,7 +232,6 @@ get_metadata(const string& path, bool follow_links) { return md; } } - if(gkfs::config::proxy::fwd_stat && CTX->use_proxy()) { err = gkfs::rpc::forward_stat_proxy(path, attr); } else { -- GitLab From ab3e3c0f4202aaced31ac5355054aa897f2613dc Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Mon, 15 Jul 2024 18:38:00 +0200 Subject: [PATCH 7/7] Fix broken CMake formatting --- CMakeLists.txt | 72 +++++----- include/client/CMakeLists.txt | 98 ++++++------- include/common/CMakeLists.txt | 4 +- include/daemon/CMakeLists.txt | 24 ++-- src/client/CMakeLists.txt | 154 ++++++++++----------- src/common/CMakeLists.txt | 36 ++--- src/common/arithmetic/CMakeLists.txt | 4 +- src/daemon/CMakeLists.txt | 76 +++++----- src/daemon/backend/data/CMakeLists.txt | 10 +- src/daemon/backend/metadata/CMakeLists.txt | 64 ++++----- 10 files changed, 271 insertions(+), 271 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a8b9cd430..a7f6b347d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,7 +84,7 @@ if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/.git) OUTPUT_VARIABLE GIT_VERSION ERROR_VARIABLE GIT_ERR OUTPUT_STRIP_TRAILING_WHITESPACE - ) + ) if ("${GIT_RET}" STREQUAL "0") string(REGEX MATCH "^v([0-9]+)\.([0-9]+)\.([0-9]+)-([0-9]+)-(.*)$" @@ -165,7 +165,7 @@ find_package(Syscall_intercept REQUIRED) if (GKFS_ENABLE_AGIOS) message(STATUS "[${PROJECT_NAME}] Checking for Agios") find_package(AGIOS REQUIRED) -endif() +endif () if (GKFS_ENABLE_CLIENT_METRICS) ### zeromq: required for sending client metrics to FTIO @@ -174,15 +174,15 @@ if (GKFS_ENABLE_CLIENT_METRICS) endif () ### Metadata backends -if(GKFS_ENABLE_ROCKSDB) +if (GKFS_ENABLE_ROCKSDB) message(STATUS "[${PROJECT_NAME}] Checking for RocksDB") add_compile_definitions(GKFS_ENABLE_ROCKSDB) find_package(RocksDB REQUIRED) message(STATUS "[${PROJECT_NAME}] RocksDB version ${RocksDB_VERSION}") message(STATUS "[${PROJECT_NAME}] RocksDB location ${RocksDB_DIR}") -endif() +endif () -if(GKFS_ENABLE_PARALLAX) +if (GKFS_ENABLE_PARALLAX) add_compile_definitions(GKFS_ENABLE_PARALLAX) # The current version of Parallax has a direct dependency on libyaml and librt. # Make sure that we find them and add them as link dependencies.. @@ -196,15 +196,15 @@ if(GKFS_ENABLE_PARALLAX) message(STATUS "[${PROJECT_NAME}] Checking for Parallax") find_package(Parallax REQUIRED) target_link_libraries(Parallax::parallax INTERFACE yaml AIO::AIO) -endif() +endif () ### Prometheus-cpp: required for the collection of GekkoFS stats ### (these expose the prometheus-cpp::pull, prometheus-cpp::push, ### prometheus-cpp::core, and curl imported targets -if(GKFS_ENABLE_PROMETHEUS) +if (GKFS_ENABLE_PROMETHEUS) find_package(CURL 7.68.0 REQUIRED) find_package(prometheus-cpp REQUIRED) # >= 1.0.0 -endif() +endif () ### Other stuff that can be found out using find_package: @@ -226,10 +226,10 @@ set(GKFS_DEPENDENCIES_PATH ${CMAKE_SOURCE_DIR}/external) ### {fmt}: required for sensible output formatting set(FMT_INSTALL OFF) include_from_source(fmt - MESSAGE "[${PROJECT_NAME}] Searching for {fmt}" - SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/fmt - GIT_REPOSITORY https://github.com/fmtlib/fmt - GIT_TAG e57ca2e3685b160617d3d95fcd9e789c4e06ca88 # v10.1.0 + MESSAGE "[${PROJECT_NAME}] Searching for {fmt}" + SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/fmt + GIT_REPOSITORY https://github.com/fmtlib/fmt + GIT_TAG e57ca2e3685b160617d3d95fcd9e789c4e06ca88 # v10.1.0 ) # ensure that fmt is linked as PIC @@ -237,19 +237,19 @@ set_property(TARGET fmt PROPERTY POSITION_INDEPENDENT_CODE ON) ### spdlog: required for logging include_from_source(spdlog - MESSAGE "[${PROJECT_NAME}] Searching for spdlog" - SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/spdlog - GIT_REPOSITORY https://github.com/gabime/spdlog.git - GIT_TAG eb3220622e73a4889eee355ffa37972b3cac3df5 # v1.9.2 - ) + MESSAGE "[${PROJECT_NAME}] Searching for spdlog" + SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/spdlog + GIT_REPOSITORY https://github.com/gabime/spdlog.git + GIT_TAG eb3220622e73a4889eee355ffa37972b3cac3df5 # v1.9.2 +) ### CLI11: used for parsing command-line options include_from_source(cli11 - MESSAGE "[${PROJECT_NAME}] Searching for CLI11" - SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/CLI11 - GIT_REPOSITORY https://github.com/CLIUtils/CLI11 - GIT_TAG v2.2.0 - ) + MESSAGE "[${PROJECT_NAME}] Searching for CLI11" + SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/CLI11 + GIT_REPOSITORY https://github.com/CLIUtils/CLI11 + GIT_TAG v2.2.0 +) if (GKFS_ENABLE_CLIENT_METRICS) ### MessagePack: used for monitoring information on the client @@ -264,9 +264,9 @@ endif () ################################################################################ ## Check configured variables/options and act accordingly ################################################################################ -if(GKFS_BUILD_DOCUMENTATION) +if (GKFS_BUILD_DOCUMENTATION) add_subdirectory(docs) -endif() +endif () if (GKFS_SYMLINK_SUPPORT) add_definitions(-DHAS_SYMLINKS) @@ -287,30 +287,30 @@ if (GKFS_RENAME_SUPPORT) add_definitions(-DHAS_RENAME) endif () -if(GKFS_MAX_INTERNAL_FDS) +if (GKFS_MAX_INTERNAL_FDS) add_definitions(-DGKFS_MAX_INTERNAL_FDS=${GKFS_MAX_INTERNAL_FDS}) -endif() +endif () -if(GKFS_MAX_OPEN_FDS) +if (GKFS_MAX_OPEN_FDS) add_definitions(-DGKFS_MAX_OPEN_FDS=${GKFS_MAX_OPEN_FDS}) -endif() +endif () -if(GKFS_ENABLE_CLIENT_LOG) +if (GKFS_ENABLE_CLIENT_LOG) add_definitions(-DGKFS_ENABLE_LOGGING) add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${GKFS_CLIENT_LOG_MESSAGE_SIZE}) endif () -if(GKFS_ENABLE_UNUSED_FUNCTIONS) +if (GKFS_ENABLE_UNUSED_FUNCTIONS) add_definitions(-DGKFS_ENABLE_UNUSED_FUNCTIONS) endif () -if(GKFS_ENABLE_PROMETHEUS) +if (GKFS_ENABLE_PROMETHEUS) add_definitions(-DGKFS_ENABLE_PROMETHEUS) endif () configure_file(include/common/cmake_configure.hpp.in include/common/cmake_configure.hpp) -if(GKFS_ENABLE_CLIENT_LOG) +if (GKFS_ENABLE_CLIENT_LOG) option(HERMES_LOGGING "" ON) option(HERMES_LOGGING_FMT_USE_BUNDLED "" OFF) option(HERMES_LOGGING_FMT_HEADER_ONLY "" OFF) @@ -369,15 +369,15 @@ if (GKFS_BUILD_TESTS) message(STATUS "[gekkofs] Check for guided distributor tests...") if (GKFS_USE_GUIDED_DISTRIBUTION) set(GKFS_TESTS_GUIDED_DISTRIBUTION "ON" CACHE STRING "Enable guided distributor tests (default: OFF)") - else() + else () set(GKFS_TESTS_GUIDED_DISTRIBUTION "OFF" CACHE STRING "Enable guided distributor tests (default: OFF)") - endif() + endif () message(STATUS "[gekkofs] Guided distributor tests: ${GKFS_TESTS_GUIDED_DISTRIBUTION}") add_subdirectory(tests) -else() +else () unset(GKFS_TESTS_INTERFACE CACHE) -endif() +endif () ################################################################################ ## Print GekkoFS configuration summary diff --git a/include/client/CMakeLists.txt b/include/client/CMakeLists.txt index d76a24a99..6d3594b0a 100644 --- a/include/client/CMakeLists.txt +++ b/include/client/CMakeLists.txt @@ -29,58 +29,58 @@ # common sources to both gkfs_intercept and gkfwd_intercept target_sources( - gkfs_intercept - PUBLIC gkfs_functions.hpp - env.hpp - hooks.hpp - intercept.hpp - logging.hpp - make_array.hpp - open_file_map.hpp - open_dir.hpp - path.hpp - preload.hpp - preload_context.hpp - preload_util.hpp - cache.hpp - rpc/rpc_types.hpp - rpc/forward_management.hpp - rpc/forward_metadata.hpp - rpc/forward_data.hpp - syscalls/args.hpp - syscalls/decoder.hpp - syscalls/errno.hpp - syscalls/rets.hpp - syscalls/syscall.hpp - syscalls/detail/syscall_info.h + gkfs_intercept + PUBLIC gkfs_functions.hpp + env.hpp + hooks.hpp + intercept.hpp + logging.hpp + make_array.hpp + open_file_map.hpp + open_dir.hpp + path.hpp + preload.hpp + preload_context.hpp + preload_util.hpp + cache.hpp + rpc/rpc_types.hpp + rpc/forward_management.hpp + rpc/forward_metadata.hpp + rpc/forward_data.hpp + syscalls/args.hpp + syscalls/decoder.hpp + syscalls/errno.hpp + syscalls/rets.hpp + syscalls/syscall.hpp + syscalls/detail/syscall_info.h ) target_sources( - gkfs_user_lib - PUBLIC gkfs_functions.hpp - env.hpp - hooks.hpp - intercept.hpp - logging.hpp - make_array.hpp - open_file_map.hpp - open_dir.hpp - path.hpp - preload.hpp - preload_context.hpp - preload_util.hpp + gkfs_user_lib + PUBLIC gkfs_functions.hpp + env.hpp + hooks.hpp + intercept.hpp + logging.hpp + make_array.hpp + open_file_map.hpp + open_dir.hpp + path.hpp + preload.hpp + preload_context.hpp + preload_util.hpp cache.hpp - rpc/rpc_types.hpp - rpc/forward_management.hpp - rpc/forward_metadata.hpp - rpc/forward_data.hpp + rpc/rpc_types.hpp + rpc/forward_management.hpp + rpc/forward_metadata.hpp + rpc/forward_data.hpp rpc/forward_malleability.hpp - syscalls/args.hpp - syscalls/decoder.hpp - syscalls/errno.hpp - syscalls/rets.hpp - syscalls/syscall.hpp - syscalls/detail/syscall_info.h - void_syscall_intercept.hpp - user_functions.hpp + syscalls/args.hpp + syscalls/decoder.hpp + syscalls/errno.hpp + syscalls/rets.hpp + syscalls/syscall.hpp + syscalls/detail/syscall_info.h + void_syscall_intercept.hpp + user_functions.hpp ) diff --git a/include/common/CMakeLists.txt b/include/common/CMakeLists.txt index 73861334e..8e23d110f 100644 --- a/include/common/CMakeLists.txt +++ b/include/common/CMakeLists.txt @@ -27,8 +27,8 @@ ################################################################################ target_sources( - gkfs_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp rpc/rpc_types.hpp - rpc/rpc_util.hpp + gkfs_daemon PUBLIC cmake_configure.hpp.in common_defs.hpp rpc/rpc_types.hpp + rpc/rpc_util.hpp ) target_sources(gkfs_proxy diff --git a/include/daemon/CMakeLists.txt b/include/daemon/CMakeLists.txt index 977c51cbd..2eb53c12a 100644 --- a/include/daemon/CMakeLists.txt +++ b/include/daemon/CMakeLists.txt @@ -27,19 +27,19 @@ ################################################################################ target_sources( - gkfs_daemon - PUBLIC daemon.hpp - util.hpp - ops/data.hpp - ops/metadentry.hpp - classes/fs_data.hpp - classes/rpc_data.hpp - handler/rpc_defs.hpp - handler/rpc_util.hpp + gkfs_daemon + PUBLIC daemon.hpp + util.hpp + ops/data.hpp + ops/metadentry.hpp + classes/fs_data.hpp + classes/rpc_data.hpp + handler/rpc_defs.hpp + handler/rpc_util.hpp malleability/malleable_manager.hpp malleability/rpc/forward_redistribution.hpp ) -if(GKFS_ENABLE_AGIOS) - target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp) -endif() +if (GKFS_ENABLE_AGIOS) + target_sources(gkfwd_daemon PUBLIC scheduler/agios.hpp) +endif () diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 2622fd57a..333c66269 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -32,110 +32,110 @@ # based on syscall interception. # ############################################################################## -add_library (gkfs_intercept SHARED) -add_library (gkfs_user_lib SHARED) +add_library(gkfs_intercept SHARED) +add_library(gkfs_user_lib SHARED) target_sources(gkfs_intercept - PRIVATE gkfs_functions.cpp - intercept.cpp - hooks.cpp - logging.cpp - open_file_map.cpp - open_dir.cpp - path.cpp - preload.cpp - preload_context.cpp - preload_util.cpp - cache.cpp - rpc/rpc_types.cpp - rpc/forward_data.cpp - rpc/forward_data_proxy.cpp - rpc/forward_management.cpp - rpc/forward_metadata.cpp - rpc/forward_metadata_proxy.cpp - syscalls/detail/syscall_info.c) + PRIVATE gkfs_functions.cpp + intercept.cpp + hooks.cpp + logging.cpp + open_file_map.cpp + open_dir.cpp + path.cpp + preload.cpp + preload_context.cpp + preload_util.cpp + cache.cpp + rpc/rpc_types.cpp + rpc/forward_data.cpp + rpc/forward_data_proxy.cpp + rpc/forward_management.cpp + rpc/forward_metadata.cpp + rpc/forward_metadata_proxy.cpp + syscalls/detail/syscall_info.c) target_sources( - gkfs_user_lib - PRIVATE gkfs_functions.cpp - intercept.cpp - hooks.cpp - logging.cpp - open_file_map.cpp - open_dir.cpp - path.cpp - preload.cpp - preload_context.cpp - preload_util.cpp - malleability.cpp - cache.cpp - rpc/rpc_types.cpp - rpc/forward_data.cpp - rpc/forward_data_proxy.cpp - rpc/forward_management.cpp - rpc/forward_metadata.cpp - rpc/forward_metadata_proxy.cpp - rpc/forward_malleability.cpp - syscalls/detail/syscall_info.c syscalls/util.S + gkfs_user_lib + PRIVATE gkfs_functions.cpp + intercept.cpp + hooks.cpp + logging.cpp + open_file_map.cpp + open_dir.cpp + path.cpp + preload.cpp + preload_context.cpp + preload_util.cpp + malleability.cpp + cache.cpp + rpc/rpc_types.cpp + rpc/forward_data.cpp + rpc/forward_data_proxy.cpp + rpc/forward_management.cpp + rpc/forward_metadata.cpp + rpc/forward_metadata_proxy.cpp + rpc/forward_malleability.cpp + syscalls/detail/syscall_info.c syscalls/util.S ) target_compile_definitions(gkfs_user_lib PUBLIC BYPASS_SYSCALL) target_link_options(gkfs_user_lib PRIVATE -z noexecstack) -if(GKFS_ENABLE_AGIOS) - target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) -endif() +if (GKFS_ENABLE_AGIOS) + target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_AGIOS) +endif () # Enable MSGPack metrics for intercept only target_link_libraries( - gkfs_intercept - PRIVATE metadata distributor env_util arithmetic path_util rpc_utils - PUBLIC dl - Mercury::Mercury - hermes - fmt::fmt - Threads::Threads - Syscall_intercept::Syscall_intercept + gkfs_intercept + PRIVATE metadata distributor env_util arithmetic path_util rpc_utils + PUBLIC dl + Mercury::Mercury + hermes + fmt::fmt + Threads::Threads + Syscall_intercept::Syscall_intercept ) # Enable MSGPack metrics for intercept only if (GKFS_ENABLE_CLIENT_METRICS) - target_link_libraries( - gkfs_intercept - PUBLIC - msgpack_util - ) - target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_CLIENT_METRICS) + target_link_libraries( + gkfs_intercept + PUBLIC + msgpack_util + ) + target_compile_definitions(gkfs_intercept PUBLIC GKFS_ENABLE_CLIENT_METRICS) endif () target_link_libraries( - gkfs_user_lib - PRIVATE metadata distributor env_util arithmetic path_util rpc_utils - PUBLIC dl - Mercury::Mercury - hermes - fmt::fmt - Threads::Threads + gkfs_user_lib + PRIVATE metadata distributor env_util arithmetic path_util rpc_utils + PUBLIC dl + Mercury::Mercury + hermes + fmt::fmt + Threads::Threads ) install( - TARGETS gkfs_intercept - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs + TARGETS gkfs_intercept + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) -set_target_properties(gkfs_user_lib - PROPERTIES - PUBLIC_HEADER "../../include/client/void_syscall_intercept.hpp" - PUBLIC_HEADER "../../include/client/user_functions.hpp" +set_target_properties(gkfs_user_lib + PROPERTIES + PUBLIC_HEADER "../../include/client/void_syscall_intercept.hpp" + PUBLIC_HEADER "../../include/client/user_functions.hpp" ) install( - TARGETS gkfs_user_lib - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs + TARGETS gkfs_user_lib + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 789dd2d04..89bc6c57f 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -31,7 +31,7 @@ include(FetchContent) add_library(rpc_utils OBJECT) set_property(TARGET rpc_utils PROPERTY POSITION_INDEPENDENT_CODE ON) target_sources(rpc_utils - PRIVATE + PRIVATE rpc/rpc_util.cpp ) @@ -46,7 +46,7 @@ target_sources(distributor ${INCLUDE_DIR}/common/rpc/distributor.hpp PRIVATE ${CMAKE_CURRENT_LIST_DIR}/rpc/distributor.cpp - ) +) add_library(statistics STATIC) set_property(TARGET statistics PROPERTY POSITION_INDEPENDENT_CODE ON) @@ -55,18 +55,18 @@ target_sources(statistics ${INCLUDE_DIR}/common/statistics/stats.hpp PRIVATE ${CMAKE_CURRENT_LIST_DIR}/statistics/stats.cpp - ) +) -if(GKFS_ENABLE_PROMETHEUS) - target_link_libraries(statistics - PRIVATE - prometheus-cpp::pull - prometheus-cpp::push - prometheus-cpp::core - curl +if (GKFS_ENABLE_PROMETHEUS) + target_link_libraries(statistics + PRIVATE + prometheus-cpp::pull + prometheus-cpp::push + prometheus-cpp::core + curl ) -endif() +endif () add_library(log_util STATIC) set_property(TARGET log_util PROPERTY POSITION_INDEPENDENT_CODE ON) @@ -75,11 +75,11 @@ target_sources(log_util ${INCLUDE_DIR}/common/log_util.hpp PRIVATE ${CMAKE_CURRENT_LIST_DIR}/log_util.cpp - ) +) target_link_libraries(log_util - PUBLIC + PUBLIC spdlog::spdlog - ) +) add_library(env_util STATIC) set_property(TARGET env_util PROPERTY POSITION_INDEPENDENT_CODE ON) @@ -89,7 +89,7 @@ target_sources(env_util PRIVATE ${INCLUDE_DIR}/config.hpp ${CMAKE_CURRENT_LIST_DIR}/env_util.cpp - ) +) add_library(metadata STATIC) set_property(TARGET metadata PROPERTY POSITION_INDEPENDENT_CODE ON) @@ -98,11 +98,11 @@ target_sources(metadata ${INCLUDE_DIR}/common/metadata.hpp PRIVATE ${CMAKE_CURRENT_LIST_DIR}/metadata.cpp - ) +) target_link_libraries(metadata PRIVATE fmt::fmt - ) +) add_library(path_util STATIC) set_property(TARGET path_util PROPERTY POSITION_INDEPENDENT_CODE ON) @@ -111,7 +111,7 @@ target_sources(path_util path_util.cpp PUBLIC ${INCLUDE_DIR}/common/path_util.hpp - ) +) if (GKFS_ENABLE_CLIENT_METRICS) add_library(msgpack_util STATIC) diff --git a/src/common/arithmetic/CMakeLists.txt b/src/common/arithmetic/CMakeLists.txt index d399c33b8..2716d0bcf 100644 --- a/src/common/arithmetic/CMakeLists.txt +++ b/src/common/arithmetic/CMakeLists.txt @@ -30,9 +30,9 @@ add_library(arithmetic INTERFACE) target_sources(arithmetic INTERFACE ${INCLUDE_DIR}/common/arithmetic/arithmetic.hpp - ) +) target_include_directories(arithmetic INTERFACE ${INCLUDE_DIR}/common/arithmetic/ - ) +) diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index d71a36d54..f6c310bcd 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -37,53 +37,53 @@ pkg_check_modules(URING IMPORTED_TARGET liburing) # We need to add here any files that may have different compile definitions target_sources( - gkfs_daemon - PRIVATE daemon.cpp - handler/srv_data.cpp - ../common/rpc/rpc_util.cpp - util.cpp - ops/metadentry.cpp - ops/data.cpp - classes/fs_data.cpp - classes/rpc_data.cpp - handler/srv_metadata.cpp - handler/srv_management.cpp + gkfs_daemon + PRIVATE daemon.cpp + handler/srv_data.cpp + ../common/rpc/rpc_util.cpp + util.cpp + ops/metadentry.cpp + ops/data.cpp + classes/fs_data.cpp + classes/rpc_data.cpp + handler/srv_metadata.cpp + handler/srv_management.cpp handler/srv_malleability.cpp malleability/malleable_manager.cpp malleability/rpc/forward_redistribution.cpp - PUBLIC ${CMAKE_SOURCE_DIR}/include/config.hpp - ${CMAKE_SOURCE_DIR}/include/version.hpp.in + PUBLIC ${CMAKE_SOURCE_DIR}/include/config.hpp + ${CMAKE_SOURCE_DIR}/include/version.hpp.in ) target_link_libraries( - gkfs_daemon - PUBLIC # internal libs - metadata - metadata_backend - storage - distributor - statistics - log_util - env_util - path_util - # external libs - CLI11::CLI11 - fmt::fmt - Mercury::Mercury - Argobots::Argobots - Margo::Margo - # others - Threads::Threads + gkfs_daemon + PUBLIC # internal libs + metadata + metadata_backend + storage + distributor + statistics + log_util + env_util + path_util + # external libs + CLI11::CLI11 + fmt::fmt + Mercury::Mercury + Argobots::Argobots + Margo::Margo + # others + Threads::Threads ) -if(GKFS_ENABLE_AGIOS) - target_sources(gkfs_daemon PRIVATE scheduler/agios.cpp) - target_compile_definitions(gkfs_daemon PUBLIC GKFS_ENABLE_AGIOS) - target_link_libraries(gkfs_daemon PRIVATE AGIOS::AGIOS) -endif() +if (GKFS_ENABLE_AGIOS) + target_sources(gkfs_daemon PRIVATE scheduler/agios.cpp) + target_compile_definitions(gkfs_daemon PUBLIC GKFS_ENABLE_AGIOS) + target_link_libraries(gkfs_daemon PRIVATE AGIOS::AGIOS) +endif () -if(GKFS_ENABLE_CODE_COVERAGE) +if (GKFS_ENABLE_CODE_COVERAGE) target_code_coverage(gkfs_daemon AUTO) -endif() +endif () install(TARGETS gkfs_daemon RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/daemon/backend/data/CMakeLists.txt b/src/daemon/backend/data/CMakeLists.txt index 4bb182fb4..f23a9f1d3 100644 --- a/src/daemon/backend/data/CMakeLists.txt +++ b/src/daemon/backend/data/CMakeLists.txt @@ -28,17 +28,17 @@ add_library(data_module STATIC - ) +) target_sources(data_module PUBLIC ${INCLUDE_DIR}/daemon/backend/data/data_module.hpp PRIVATE ${CMAKE_CURRENT_LIST_DIR}/data_module.cpp - ) +) target_link_libraries(data_module - PRIVATE + PRIVATE log_util ) @@ -51,7 +51,7 @@ target_sources(storage ${INCLUDE_DIR}/common/common_defs.hpp ${INCLUDE_DIR}/daemon/backend/data/file_handle.hpp ${CMAKE_CURRENT_LIST_DIR}/chunk_storage.cpp - ) +) target_link_libraries(storage PRIVATE @@ -61,7 +61,7 @@ target_link_libraries(storage # open issue for std::filesystem https://gitlab.kitware.com/cmake/cmake/-/issues/17834 stdc++fs -ldl - ) +) #target_include_directories(storage # PRIVATE diff --git a/src/daemon/backend/metadata/CMakeLists.txt b/src/daemon/backend/metadata/CMakeLists.txt index 878663ff4..0468e7cf8 100644 --- a/src/daemon/backend/metadata/CMakeLists.txt +++ b/src/daemon/backend/metadata/CMakeLists.txt @@ -29,54 +29,54 @@ # Define metadata module library first add_library(metadata_module STATIC) target_sources( - metadata_module - PUBLIC ${CMAKE_SOURCE_DIR}/include/daemon/backend/metadata/metadata_module.hpp - PRIVATE metadata_module.cpp + metadata_module + PUBLIC ${CMAKE_SOURCE_DIR}/include/daemon/backend/metadata/metadata_module.hpp + PRIVATE metadata_module.cpp ) target_link_libraries(metadata_module PRIVATE log_util) # Define metadata_backend and its common dependencies and sources add_library(metadata_backend STATIC) target_sources( - metadata_backend - PUBLIC + metadata_backend + PUBLIC ${CMAKE_SOURCE_DIR}/include/daemon/backend/metadata/db.hpp ${CMAKE_SOURCE_DIR}/include/daemon/backend/exceptions.hpp ${CMAKE_SOURCE_DIR}/include/daemon/backend/metadata/metadata_backend.hpp - PRIVATE ${CMAKE_SOURCE_DIR}/include/daemon/backend/metadata/merge.hpp - merge.cpp db.cpp + PRIVATE ${CMAKE_SOURCE_DIR}/include/daemon/backend/metadata/merge.hpp + merge.cpp db.cpp ) target_link_libraries( - metadata_backend - PRIVATE metadata_module dl log_util path_util + metadata_backend + PRIVATE metadata_module dl log_util path_util ) -if(GKFS_ENABLE_ROCKSDB) - target_sources( - metadata_backend - PUBLIC ${CMAKE_SOURCE_DIR}/include/daemon/backend/metadata/rocksdb_backend.hpp - PRIVATE rocksdb_backend.cpp - ) +if (GKFS_ENABLE_ROCKSDB) + target_sources( + metadata_backend + PUBLIC ${CMAKE_SOURCE_DIR}/include/daemon/backend/metadata/rocksdb_backend.hpp + PRIVATE rocksdb_backend.cpp + ) - # If liburing is available in the system, RocksDB will have been built - # with liburing support. If so, we need to propagate the dependency to - # the daemon. Since liburing exports a pkg-config .pc file, we can use it - # to retrieve its details. - pkg_check_modules(URING IMPORTED_TARGET liburing) + # If liburing is available in the system, RocksDB will have been built + # with liburing support. If so, we need to propagate the dependency to + # the daemon. Since liburing exports a pkg-config .pc file, we can use it + # to retrieve its details. + pkg_check_modules(URING IMPORTED_TARGET liburing) - if(URING_FOUND) - target_link_libraries(metadata_backend PUBLIC PkgConfig::URING) - endif() + if (URING_FOUND) + target_link_libraries(metadata_backend PUBLIC PkgConfig::URING) + endif () - target_link_libraries(metadata_backend PUBLIC RocksDB::rocksdb) -endif() + target_link_libraries(metadata_backend PUBLIC RocksDB::rocksdb) +endif () -if(GKFS_ENABLE_PARALLAX) - target_sources(metadata_backend - PUBLIC ${CMAKE_SOURCE_DIR}/include/daemon/backend/metadata/parallax_backend.hpp - PRIVATE parallax_backend.cpp +if (GKFS_ENABLE_PARALLAX) + target_sources(metadata_backend + PUBLIC ${CMAKE_SOURCE_DIR}/include/daemon/backend/metadata/parallax_backend.hpp + PRIVATE parallax_backend.cpp ) - target_link_libraries(metadata_backend PUBLIC Parallax::parallax - Parallax::log) -endif() + target_link_libraries(metadata_backend PUBLIC Parallax::parallax + Parallax::log) +endif () -- GitLab