Commit a4543512 authored by Ramon Nou's avatar Ramon Nou
Browse files

feat(cas,inline): add Content-Addressed Storage deduplication and export... 

feat(cas,inline): add Content-Addressed Storage deduplication and export INLINE_DATA_SIZE via common env

- Implement CAS chunk deduplication with xxhash3 in daemon and client
- Export INLINE_DATA_SIZE via include/common/env.hpp with runtime config in daemon and client
- Add unit tests for INLINE_DATA_SIZE and CAS environment parsing
parent 714fe3aa

include/client/preload_context.hpp

+14 −0
Changes for include/client/preload_context.hpp: 14 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -152,6 +152,8 @@ private: 
    bool use_write_size_cache_{false};

    bool local_bypass_{true};

    bool has_local_host_{false};

    bool cas_enabled_{false};

    size_t cas_min_size_{16384};





    std::string cwd_;
@@ -401,6 +403,18 @@ public: 
    void

    has_local_host(bool has);



    bool

    cas_enabled() const;



    void

    cas_enabled(bool enable);



    size_t

    cas_min_size() const;



    void

    cas_min_size(size_t size);



    void

    enable_interception();

include/common/env.hpp

+3 −0
Changes for include/common/env.hpp: 3 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -78,10 +78,13 @@ static constexpr auto CREATE_CHECK_PARENTS = ADD_PREFIX("CREATE_CHECK_PARENTS"); 
static constexpr auto SYMLINK_SUPPORT = ADD_PREFIX("SYMLINK_SUPPORT");

static constexpr auto RENAME_SUPPORT = ADD_PREFIX("RENAME_SUPPORT");

static constexpr auto USE_INLINE_DATA = ADD_PREFIX("USE_INLINE_DATA");

static constexpr auto INLINE_DATA_SIZE = ADD_PREFIX("INLINE_DATA_SIZE");

static constexpr auto USE_DIRENTS_COMPRESSION =

        ADD_PREFIX("USE_DIRENTS_COMPRESSION");

static constexpr auto ENABLE_TRACE = ADD_PREFIX("ENABLE_TRACE");

static constexpr auto TRACE_SAMPLE_RATE = ADD_PREFIX("TRACE_SAMPLE_RATE");

static constexpr auto ENABLE_CAS = ADD_PREFIX("ENABLE_CAS");

static constexpr auto CAS_MIN_SIZE = ADD_PREFIX("CAS_MIN_SIZE");



} // namespace gkfs::env

include/config.hpp

+2 −1
Changes for include/config.hpp: 2 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -40,6 +40,7 @@ 
#ifndef GEKKOFS_CONFIG_HPP

#define GEKKOFS_CONFIG_HPP



#include <cstddef>

#include <common/cmake_configure.hpp>



// environment prefixes (are concatenated in env module at compile time)
@@ -141,7 +142,7 @@ inline bool rename_support = true; 
// Use inline data for small files

// Use inline data for small files

inline bool use_inline_data = true;

constexpr auto inline_data_size = 4096; // in bytes

inline size_t inline_data_size = 4096; // in bytes

// Next options may break consistency, and definitely break tests. Use with

// care. Optimize write operations for small files (files are not created until

// a write appears)

include/daemon/backend/data/chunk_storage.hpp

+18 −0
Changes for include/daemon/backend/data/chunk_storage.hpp: 18 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -85,6 +85,9 @@ private: 


    std::string root_path_; //!< Path to GekkoFS root directory

    size_t chunksize_; //!< File system chunksize. TODO Why does that exist?

    bool cas_enabled_{false};

    size_t cas_min_size_{16384};

    std::string cas_dir_;



    /**

     * @brief Converts an internal gkfs path under the root dir to the absolute
@@ -210,6 +213,21 @@ public: 


    std::string

    get_chunk_directory();



    [[nodiscard]] bool

    cas_enabled() const {

        return cas_enabled_;

    }



    void

    cas_enabled(bool enable) {

        cas_enabled_ = enable;

    }



    [[nodiscard]] size_t

    cas_min_size() const {

        return cas_min_size_;

    }

};



} // namespace gkfs::data

src/client/preload.cpp

+18 −0
Changes for src/client/preload.cpp: 18 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -758,6 +758,8 @@ init_preload() { 
                               gkfs::config::metadata::use_inline_data

                                       ? "ON"

                                       : "OFF") == "ON";

    gkfs::config::metadata::inline_data_size = static_cast<size_t>(

            gkfs::env::get_var(gkfs::env::INLINE_DATA_SIZE, 4096));

    gkfs::config::metadata::create_write_optimization =

            gkfs::env::get_var(gkfs::env::CREATE_WRITE_OPTIMIZATION,

                               gkfs::config::metadata::create_write_optimization
@@ -802,6 +804,22 @@ init_preload() { 
    CTX->local_bypass(bypass_enabled);

    LOG(INFO, "Local chunk bypass: {}", CTX->local_bypass() ? "ON" : "OFF");



    bool cas_enabled = gkfs::env::get_var_bool(gkfs::env::ENABLE_CAS, false);

    if(!cas_enabled && gkfs::env::var_is_set("LIBGKFS_ENABLE_CAS")) {

        cas_enabled = gkfs::env::get_var_bool("LIBGKFS_ENABLE_CAS", false);

    }

    CTX->cas_enabled(cas_enabled);



    size_t cas_min_size = static_cast<size_t>(

            gkfs::env::get_var(gkfs::env::CAS_MIN_SIZE, 8192));

    if(cas_min_size == 8192 && gkfs::env::var_is_set("LIBGKFS_CAS_MIN_SIZE")) {

        cas_min_size = static_cast<size_t>(

                gkfs::env::get_var("LIBGKFS_CAS_MIN_SIZE", 8192));

    }

    CTX->cas_min_size(cas_min_size);

    LOG(INFO, "Content-Addressed Deduplication (CAS): {} (min_size: {} bytes)",

        CTX->cas_enabled() ? "ON" : "OFF", CTX->cas_min_size());



#ifndef BYPASS_SYSCALL

    CTX->enable_interception();

    gkfs::preload::start_interception();