Commit b29b8445 authored by Marc Vef's avatar Marc Vef
Browse files

Adding write_size cache, enable by config and disable via... 

Adding write_size cache, enable by config and disable via `LIBGKFS_DISABLE_WRITE_SIZE_CACHE=on`. Flush happens on close/fsync

This avoids heavy metadata traffic during small I/O especially for single shared files which can overwhelm the corresponding metadata daemon
parent 9d525d46

include/client/cache.hpp

+23 −2
Original line number Diff line number Diff line
@@ -39,6 +39,7 @@ 
#include <mutex>

#include <optional>

#include <cstdint>

#include <utility>



namespace gkfs::cache {
@@ -89,8 +90,28 @@ public: 
};

} // namespace dir



//// <path<cnt, size>>

// std::unordered_map<std::string, std::pair<size_t, size_t>> size_cache;

namespace file {

class WriteSizeCache {

private:

    // <path<cnt, size>>

    std::unordered_map<std::string, std::pair<size_t, size_t>> size_cache;

    std::mutex mtx_;



public:

    WriteSizeCache() = default;



    virtual ~WriteSizeCache() = default;



    std::pair<size_t, size_t>

    record_write_size(std::string path, size_t size);



    void

    reset_write_size(const std::string& path);



    std::optional<size_t>

    remove_write_size(const std::string& path);

};

} // namespace file





} // namespace gkfs::cache

include/client/env.hpp

+2 −0
Original line number Diff line number Diff line
@@ -61,6 +61,8 @@ static constexpr auto METRICS_IP_PORT = ADD_PREFIX("METRICS_IP_PORT"); 
static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL");

static constexpr auto PROXY_PID_FILE = ADD_PREFIX("PROXY_PID_FILE");

static constexpr auto DISABLE_DENTRY_CACHE = ADD_PREFIX("DISABLE_DENTRY_CACHE");

static constexpr auto DISABLE_WRITE_SIZE_CACHE =

        ADD_PREFIX("DISABLE_WRITE_SIZE_CACHE");



} // namespace gkfs::env

include/client/gkfs_functions.hpp

+3 −0
Original line number Diff line number Diff line
@@ -153,6 +153,9 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp, 
int

gkfs_rmdir(const std::string& path);



int

gkfs_fsync(unsigned int fd);



int

gkfs_close(unsigned int fd);

include/client/preload_context.hpp

+15 −0
Original line number Diff line number Diff line
@@ -101,6 +101,9 @@ private: 
    std::shared_ptr<FsConfig> fs_conf_;

    std::shared_ptr<gkfs::cache::dir::DentryCache> dentry_cache_;

    bool use_dentry_cache_{false};

    std::shared_ptr<gkfs::cache::file::WriteSizeCache> write_size_cache_;

    bool use_write_size_cache_{false};





    std::string cwd_;

    std::vector<std::string> mountdir_components_;
@@ -252,6 +255,18 @@ public: 
    void

    use_dentry_cache(bool use_dentry_cache);



    std::shared_ptr<gkfs::cache::file::WriteSizeCache>

    write_size_cache() const;



    void

    write_size_cache(std::shared_ptr<gkfs::cache::file::WriteSizeCache>

                             write_size_cache);



    bool

    use_write_size_cache() const;



    void

    use_write_size_cache(bool use_dentry_cache);



    void

    enable_interception();

src/client/cache.cpp

+43 −0
Original line number Diff line number Diff line
@@ -123,4 +123,47 @@ DentryCache::clear() { 


} // namespace dir



namespace file {



std::pair<size_t, size_t>

WriteSizeCache::record_write_size(std::string path, size_t size) {

    std::lock_guard<std::mutex> const lock(mtx_);

    auto& pair = size_cache.try_emplace(std::move(path), std::make_pair(0, 0))

                         .first->second;

    pair.first++;

    if(pair.second < size) {

        pair.second = size;

    }

    return pair;

}



void

WriteSizeCache::reset_write_size(const std::string& path) {

    std::lock_guard<std::mutex> const lock(mtx_);

    auto it = size_cache.find(path);

    if(it == size_cache.end()) {

        return;

    }

    // reset cnt

    it->second.first = 0;

}



std::optional<size_t>

WriteSizeCache::remove_write_size(const std::string& path) {

    std::lock_guard<std::mutex> const lock(mtx_);

    auto it = size_cache.find(path);

    if(it == size_cache.end()) {

        return {};

    }

    auto entry = it->second;

    size_cache.erase(it);

    // no new updates in cache, don't return size

    if(entry.first == 0) {

        return {};

    }

    return entry.second;

}



} // namespace file



} // namespace gkfs::cache