Commit ea91055c authored by David Auer's avatar David Auer
Browse files

Relocate chunks

parent b2a4317e

include/daemon/backend/data/chunk_storage.hpp

+3 −0
Original line number Diff line number Diff line
@@ -79,6 +79,9 @@ public: 
    trim_chunk_space(const std::string& file_path,

                     gkfs::rpc::chnk_id_t chunk_start);



    void

    remove_chunk(const std::string& file_path, gkfs::rpc::chnk_id_t chunk_id);



    void

    truncate_chunk_file(const std::string& file_path,

                        gkfs::rpc::chnk_id_t chunk_id, off_t length);

include/daemon/handler/rpc_defs.hpp

+2 −0
Original line number Diff line number Diff line
@@ -50,6 +50,8 @@ DECLARE_MARGO_RPC_HANDLER(rpc_srv_relocation_start) 


DECLARE_MARGO_RPC_HANDLER(rpc_srv_relocate_metadata)



DECLARE_MARGO_RPC_HANDLER(rpc_srv_relocate_chunk)





// data

DECLARE_MARGO_RPC_HANDLER(rpc_srv_remove_data)

include/global/global_defs.hpp

+1 −0
Original line number Diff line number Diff line
@@ -38,6 +38,7 @@ constexpr auto mk_symlink = "rpc_srv_mk_symlink"; 
#endif

constexpr auto relocation_start = "rpc_srv_relocation_start";

constexpr auto relocate_metadata = "rpc_srv_relocate_metadata";

constexpr auto relocate_chunk = "rpc_srv_relocate_chunk";



constexpr auto write = "rpc_srv_write_data";

constexpr auto read = "rpc_srv_read_data";

include/global/rpc/rpc_types.hpp

+12 −6
Original line number Diff line number Diff line
@@ -100,16 +100,22 @@ MERCURY_GEN_PROC(rpc_config_out_t, 
                         (hg_uint32_t)(gid)))





MERCURY_GEN_PROC(rpc_relocation_start_in_t, ((hg_uint64_t)(host_id)))



MERCURY_GEN_PROC(rpc_relocate_metadata_in_t,

                 ((hg_const_string_t)(key))((hg_const_string_t)(value)))





MERCURY_GEN_PROC(rpc_chunk_stat_in_t, ((hg_int32_t)(dummy)))



MERCURY_GEN_PROC(rpc_chunk_stat_out_t,

                 ((hg_int32_t)(err))((hg_uint64_t)(chunk_size))(

                         (hg_uint64_t)(chunk_total))((hg_uint64_t)(chunk_free)))





// dynamic data placement: management

MERCURY_GEN_PROC(rpc_relocation_start_in_t, ((hg_uint64_t)(host_id)))



// dynamic data placement: relocation

MERCURY_GEN_PROC(rpc_relocate_metadata_in_t,

                 ((hg_const_string_t)(key))((hg_const_string_t)(value)))



MERCURY_GEN_PROC(rpc_relocate_chunk_in_t,

                 ((hg_const_string_t)(path))((hg_uint64_t)(chunk_id))(

                         (hg_bulk_t)(bulk_handle)))



#endif // LFS_RPC_TYPES_HPP

src/daemon/backend/data/chunk_storage.cpp

+32 −13
Original line number Diff line number Diff line
@@ -46,19 +46,6 @@ ChunkStorage::get_chunks_dir(const string& file_path) { 
    return chunk_dir;

}



/**

 * Reverse function of get_chunks_dir.

 * @param chunks_dir The directory name in the backend (not a path).

 * @returns The absolute path of the file inside GekkoFS.

 */

string

ChunkStorage::get_file_path(const string& chunks_dir) {

    assert(chunks_dir.find('/') == string::npos);

    string file_path = "/"s + chunks_dir;

    ::replace(file_path.begin(), file_path.end(), ':', '/');

    return file_path;

}



string

ChunkStorage::get_chunk_path(const string& file_path,

                             gkfs::rpc::chnk_id_t chunk_id) {
@@ -85,6 +72,19 @@ ChunkStorage::init_chunk_space(const string& file_path) const { 


// public functions



/**

 * Reverse function of get_chunks_dir.

 * @param chunks_dir The directory name in the backend (not a path).

 * @returns The absolute path of the file inside GekkoFS.

 */

string

ChunkStorage::get_file_path(const string& chunks_dir) {

    assert(chunks_dir.find('/') == string::npos);

    string file_path = "/"s + chunks_dir;

    ::replace(file_path.begin(), file_path.end(), ':', '/');

    return file_path;

}



/**

 *

 * @param path
@@ -304,6 +304,25 @@ ChunkStorage::trim_chunk_space(const string& file_path, 
                        __func__, file_path));

}



/**

 * Delete a single chunk. "No such file" is silently ignored.

 * @param file_path

 * @param chunk_id

 * @throws ChunkStorageException

 */

void

ChunkStorage::remove_chunk(const string& file_path,

                           gkfs::rpc::chnk_id_t chunk_id) {

    auto chunk_path = absolute(get_chunk_path(file_path, chunk_id));

    auto err = unlink(chunk_path.c_str());

    if(err == -1 && errno != ENOENT) {

        throw ChunkStorageException(

                EIO,

                fmt::format("{}() Error occurred when removing chunk '{}/{}'",

                            __func__, file_path, chunk_id));

    }

}



/**

 * Truncates a single chunk file to a given length

 * @param file_path