From ec41c156c23dd8e0133da3dd1e990fd962c4b619 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 12 May 2025 12:03:48 +0200 Subject: [PATCH] GENERATOR/CONSUMER env locking feature --- CHANGELOG.md | 2 + README.md | 5 ++ include/client/env.hpp | 4 ++ include/client/preload_context.hpp | 14 ++++ src/client/gkfs_functions.cpp | 110 ++++++++++++++++++++++++++++- src/client/preload.cpp | 9 +++ src/client/preload_context.cpp | 21 ++++++ 7 files changed, 162 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 900ab19aa..37f104261 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Tests to cover proxy and (malleability) ([!222](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/222)) - New fd generation method ([!225](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/202)) - Use LIBGKFS_PROTECT_FD=1 to enable the original method of assignation and protection. + - Lock system (server level) ([!245](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/245)) + - Use PROTECT_FILES_GENERATOR=1 and PROTECT_FILES_CONSUMER=1 to enable. Generator, creates transparent .lockgekko files that blocks the open (for some seconds) of any consumer. Multiple opens / closes for generator are managed. ### Changed - Tests check ret for -1 instead of 10000 fd ([!320](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/320)) diff --git a/README.md b/README.md index 03d89091e..ee5eab21c 100644 --- a/README.md +++ b/README.md @@ -589,6 +589,11 @@ until the file is closed. The cache does not impact the consistency of the file When the user creates a fd, this is protected from normal fds with a recolocation. This theoretically protects the fd from being closed from outside. However a new fd assignation system has been developed and is activated by default. - `LIBGKFS_PROTECT_FD=1` - Enable the original method of assignation and protection. +##### Lightweight File-Locking (server side) +Using two environment variables +- `LIBGKFS_PROTECT_FILES_GENERATOR=1` enables the application as generator, so a open will create (and increase) a file .lockgekko and close will remove it (or decrease its value). The behaviour only uses metadata at server side. +- `LIBGKFS_PROTECT_FILES_CONSUMER=1` enables the application as consumer, so a open will wait until the .lockgekko dissapears. The wait is limited to ~40 seconds. + ### Daemon #### Logging - `GKFS_DAEMON_LOG_PATH` - Path to the log file of the daemon. diff --git a/include/client/env.hpp b/include/client/env.hpp index ec400aba4..5140d8f11 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -69,6 +69,10 @@ static constexpr auto METRICS_IP_PORT = ADD_PREFIX("METRICS_IP_PORT"); #endif static constexpr auto PROTECT_FD = ADD_PREFIX("PROTECT_FD"); +static constexpr auto PROTECT_FILES_GENERATOR = + ADD_PREFIX("PROTECT_FILES_GENERATOR"); +static constexpr auto PROTECT_FILES_CONSUMER = + ADD_PREFIX("PROTECT_FILES_CONSUMER"); static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL"); static constexpr auto PROXY_PID_FILE = ADD_PREFIX("PROXY_PID_FILE"); namespace cache { diff --git a/include/client/preload_context.hpp b/include/client/preload_context.hpp index 9f4b38875..245e15c28 100644 --- a/include/client/preload_context.hpp +++ b/include/client/preload_context.hpp @@ -140,6 +140,8 @@ private: int replicas_; bool protect_fds_{false}; + bool protect_files_generator_{false}; + bool protect_files_consumer_{false}; std::shared_ptr write_metrics_; std::shared_ptr read_metrics_; @@ -322,6 +324,18 @@ public: void protect_fds(bool protect); + bool + protect_files_generator() const; + + void + protect_files_generator(bool protect); + + bool + protect_files_consumer() const; + + void + protect_files_consumer(bool protect); + const std::shared_ptr write_metrics(); diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 3ca3dce19..1b338fc0a 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -48,6 +48,8 @@ #include #include #include +#include +#include #include #ifdef GKFS_ENABLE_CLIENT_METRICS @@ -145,6 +147,65 @@ check_parent_dir(const std::string& path) { namespace gkfs::syscall { + +/** + * @brief generate_lock_file + * @param path + * @param increase + * + * Creates, if it does not exist, a lock file, path+".lockgekko", empty + * If increase is true, increase the size of the file +1 + * if increase is false, decrease the size of the file -1 + * If size == 0, delete the file + * Using calls : forward_create, forward_stat, forward_remove, forward_decr_size + * and forward_update_metadentry_size Proxy not supported + */ +void +generate_lock_file(const std::string& path, bool increase) { + auto lock_path = path + ".lockgekko"; + if(increase) { + auto md = gkfs::utils::get_metadata(lock_path); + if(!md) { + gkfs::rpc::forward_create(lock_path, 0777 | S_IFREG, 0); + } + gkfs::rpc::forward_update_metadentry_size(lock_path, 1, 0, false, 0); + } else { + auto md = gkfs::utils::get_metadata(lock_path); + if(md) { + if(md.value().size() == 1) { + LOG(DEBUG, "Deleting Lock file {}", lock_path); + gkfs::rpc::forward_remove(lock_path, false, 0); + } else { + gkfs::rpc::forward_decr_size(lock_path, md.value().size() - 1, + 0); + } + } + } +} + +/** + * @brief generate_lock_file + * @param path + * + * Test if the lock file exists, if it exists, wait 0.5 second and check again + * (max 80 times) Using calls : forward_stat + */ +void +test_lock_file(const std::string& path) { + auto lock_path = path + ".lockgekko"; + auto md = gkfs::utils::get_metadata(lock_path); + if(md) { + LOG(DEBUG, "Lock file exists {} --> {}", lock_path, md->size()); + for(int i = 0; i < 80; i++) { + if(!md) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + md = gkfs::utils::get_metadata(lock_path); + } + } +} + /** * gkfs wrapper for open() system calls * errno may be set @@ -197,9 +258,14 @@ gkfs_open(const std::string& path, mode_t mode, int flags) { return -1; } } else { - // file was successfully created. Add to filemap - return CTX->file_map()->add( + auto fd = CTX->file_map()->add( std::make_shared(path, flags)); + // CREATE_MODE + if(CTX->protect_files_generator()) { + generate_lock_file(path, true); + } + // file was successfully created. Add to filemap + return fd; } } else { auto md_ = gkfs::utils::get_metadata(path); @@ -277,8 +343,18 @@ gkfs_open(const std::string& path, mode_t mode, int flags) { } } - return CTX->file_map()->add( + auto fd = CTX->file_map()->add( std::make_shared(path, flags)); + + + if(CTX->protect_files_consumer()) { + test_lock_file(path); + } + + if(CTX->protect_files_generator()) { + generate_lock_file(path, true); + } + return fd; } /** @@ -1480,6 +1556,14 @@ gkfs_getdents(unsigned int fd, struct linux_dirent* dirp, unsigned int count) { while(pos < open_dir->size()) { // get dentry fir current position auto de = open_dir->getdent(pos); + if(CTX->protect_files_consumer() or CTX->protect_files_generator()) { + // if de.name ends with lockgekko jump to the next file + if(de.name().size() >= 10 && + de.name().substr(de.name().size() - 10) == ".lockgekko") { + pos++; + continue; + } + } /* * Calculate the total dentry size within the kernel struct * `linux_dirent` depending on the file name size. The size is then @@ -1549,6 +1633,14 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, struct linux_dirent64* current_dirp = nullptr; while(pos < open_dir->size()) { auto de = open_dir->getdent(pos); + if(CTX->protect_files_consumer() or CTX->protect_files_generator()) { + // if de.name ends with lockgekko jump to the next file + if(de.name().size() >= 10 && + de.name().substr(de.name().size() - 10) == ".lockgekko") { + pos++; + continue; + } + } /* * Calculate the total dentry size within the kernel struct * `linux_dirent` depending on the file name size. The size is then @@ -1644,6 +1736,11 @@ gkfs_close(unsigned int fd) { CTX->file_map()->get(fd)->path()); } } + + if(CTX->protect_files_generator()) { + auto path = CTX->file_map()->get(fd)->path(); + generate_lock_file(path, false); + } // No call to the daemon is required CTX->file_map()->remove(fd); return 0; @@ -1764,6 +1861,13 @@ gkfs_get_file_list(const std::string& path) { while(pos < open_dir->size()) { auto de = open_dir->getdent(pos++); + if(CTX->protect_files_consumer() or CTX->protect_files_generator()) { + // if de.name ends with lockgekko jump to the next file + if(de.name().size() >= 10 && + de.name().substr(de.name().size() - 10) == ".lockgekko") { + continue; + } + } file_list.push_back(de.name()); } return file_list; diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 2950de7a8..3c55090c0 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -239,6 +239,15 @@ init_environment() { "Failed to connect to hosts: "s + e.what()); } + CTX->protect_files_generator( + gkfs::env::get_var(gkfs::env::PROTECT_FILES_GENERATOR, 0)); + + CTX->protect_files_consumer( + gkfs::env::get_var(gkfs::env::PROTECT_FILES_CONSUMER, 0)); + + LOG(INFO, "Lock-Files : Generator = {} / Consumer = {}", + CTX->protect_files_generator(), CTX->protect_files_consumer()); + /* Setup distributor */ auto forwarding_map_file = gkfs::env::get_var( gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); diff --git a/src/client/preload_context.cpp b/src/client/preload_context.cpp index 6e71ec80d..ccd5b1ed9 100644 --- a/src/client/preload_context.cpp +++ b/src/client/preload_context.cpp @@ -659,6 +659,27 @@ PreloadContext::get_replicas() { return replicas_; } +bool +PreloadContext::protect_files_generator() const { + return protect_files_generator_; +} + +void +PreloadContext::protect_files_generator(bool protect) { + protect_files_generator_ = protect; +} + +bool +PreloadContext::protect_files_consumer() const { + return protect_files_consumer_; +} + +void +PreloadContext::protect_files_consumer(bool protect) { + protect_files_consumer_ = protect; +} + + const std::shared_ptr PreloadContext::write_metrics() { return write_metrics_; -- GitLab