Verified Commit 0d52833e authored by Marc Vef's avatar Marc Vef
Browse files

Adding implicit_data_removal config for optimized remove operations

parent 6539c2cc

include/config.hpp

+7 −0
Original line number Diff line number Diff line
@@ -48,6 +48,13 @@ constexpr auto use_ctime = false; 
constexpr auto use_mtime = false;

constexpr auto use_link_cnt = false;

constexpr auto use_blocks = false;

/*

 * If true, all chunks on the same host are removed during a metadata remove

 * rpc. This is a technical optimization that reduces the number of RPCs for

 * remove operations. This setting could be useful for future asynchronous

 * remove implementations where the data should not be removed immediately.

 */

constexpr auto implicit_data_removal = true;



// metadata logic

// Check for existence of file metadata before create. This done on RocksDB

src/client/rpc/forward_metadata.cpp

+9 −6
Original line number Diff line number Diff line
@@ -169,12 +169,15 @@ forward_remove(const std::string& path) { 
            for(uint64_t chnk_id = chnk_start; chnk_id <= chnk_end; chnk_id++) {

                const auto chnk_host_id =

                        CTX->distributor()->locate_data(path, chnk_id);

                if constexpr(gkfs::config::metadata::implicit_data_removal) {

                    /*

                 * If the chnk host matches the metadata host the remove request

                 * as already been sent as part of the metadata remove request.

                     * If the chnk host matches the metadata host the remove

                     * request as already been sent as part of the metadata

                     * remove request.

                     */

                    if(chnk_host_id == metadata_host_id)

                        continue;

                }

                const auto endp_chnk = CTX->hosts().at(chnk_host_id);



                LOG(DEBUG, "Sending RPC to host: {}", endp_chnk.to_string());

src/daemon/backend/data/chunk_storage.cpp

+2 −2
Original line number Diff line number Diff line
@@ -108,8 +108,8 @@ ChunkStorage::destroy_chunk_space(const string& file_path) const { 
    try {

        // Note: remove_all does not throw an error when path doesn't exist.

        auto n = fs::remove_all(chunk_dir);

        log_->debug("{}() Removed '{}' files from '{}'", __func__, n,

                    chunk_dir);

        log_->debug("{}() Removed '{}' files and directories from '{}'",

                    __func__, n, chunk_dir);

    } catch(const fs::filesystem_error& e) {

        auto err_str = fmt::format(

                "{}() Failed to remove chunk directory. Path: '{}', Error: '{}'",

src/daemon/handler/srv_metadata.cpp

+9 −0
Original line number Diff line number Diff line
@@ -167,10 +167,19 @@ rpc_srv_remove_metadata(hg_handle_t handle) { 
        out.err = 0;

        out.mode = md.mode();

        out.size = md.size();

        if constexpr(gkfs::config::metadata::implicit_data_removal) {

            if(S_ISREG(md.mode()) && (md.size() != 0))

                GKFS_DATA->storage()->destroy_chunk_space(in.path);

        }

    } catch(const gkfs::metadata::DBException& e) {

        GKFS_DATA->spdlogger()->error("{}(): path '{}' message '{}'", __func__,

                                      in.path, e.what());

        out.err = EIO;

    } catch(const gkfs::data::ChunkStorageException& e) {

        GKFS_DATA->spdlogger()->error(

                "{}(): path '{}' errcode '{}' message '{}'", __func__, in.path,

                e.code().value(), e.what());

        out.err = e.code().value();

    } catch(const std::exception& e) {

        GKFS_DATA->spdlogger()->error("{}() path '{}' message '{}'", __func__,

                                      in.path, e.what());