Commit b2a4317e authored by David Auer's avatar David Auer
Browse files

Iterate chunks per daemon

parent 1079419e

include/daemon/backend/data/chunk_storage.hpp

+7 −0
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ 
#include <string>

#include <memory>

#include <system_error>

#include <filesystem>



/* Forward declarations */

namespace spdlog {
@@ -84,6 +85,12 @@ public: 


    ChunkStat

    chunk_stat() const;



    std::filesystem::directory_iterator

    chunks_directory_iterator();



    std::string

    get_file_path(const std::string& chunks_dir);

};



} // namespace data

src/daemon/backend/data/chunk_storage.cpp

+22 −0
Original line number Diff line number Diff line
@@ -46,6 +46,19 @@ ChunkStorage::get_chunks_dir(const string& file_path) { 
    return chunk_dir;

}



/**

 * Reverse function of get_chunks_dir.

 * @param chunks_dir The directory name in the backend (not a path).

 * @returns The absolute path of the file inside GekkoFS.

 */

string

ChunkStorage::get_file_path(const string& chunks_dir) {

    assert(chunks_dir.find('/') == string::npos);

    string file_path = "/"s + chunks_dir;

    ::replace(file_path.begin(), file_path.end(), ':', '/');

    return file_path;

}



string

ChunkStorage::get_chunk_path(const string& file_path,

                             gkfs::rpc::chnk_id_t chunk_id) {
@@ -339,4 +352,13 @@ ChunkStorage::chunk_stat() const { 
    return {chunksize_, bytes_total / chunksize_, bytes_free / chunksize_};

}



/**

 * Returns a directory_iterator of the chunk storage directory.

 * @return std::filesystem::directory_iterator

 */

fs::directory_iterator

ChunkStorage::chunks_directory_iterator() {

    return fs::directory_iterator(root_path_);

}



} // namespace gkfs::data
 
 No newline at end of file

src/daemon/relocation/transmitter.cpp

+37 −8
Original line number Diff line number Diff line
@@ -15,12 +15,15 @@ 


#include <daemon/daemon.hpp>

#include <daemon/backend/metadata/db.hpp>

#include <daemon/backend/data/chunk_storage.hpp>

#include <daemon/handler/rpc_defs.hpp>

#include <global/hostsfile_util.hpp>

#include <daemon/relocation/config_manager.hpp>

#include <global/rpc/rpc_types.hpp>



#include <iostream>

#include <filesystem>

namespace fs = std::filesystem;



#include <fmt/format.h>
@@ -46,14 +49,16 @@ transmit_metadata(gkfs::rpc::host_t localhost) { 


    // Relocate metadata



    for(auto metadatum : GKFS_DATA->mdb()->get_all()) {

        if(metadatum.first == "/") {

    for(auto metadentry : GKFS_DATA->mdb()->get_all()) {

        if(metadentry.first == "/") {

            continue;

        }

        gkfs::rpc::host_t destination =

                distributor.locate_file_metadata(metadatum.first);

        cout << "Checking " << metadatum.first << " Val: " << metadatum.second

             << " Move: " << (destination != localhost) << "\n";

        auto destination = distributor.locate_file_metadata(metadentry.first);

        cout << "Metadentry " << metadentry.first << " : " << metadentry.second

             << (destination == localhost

                         ? " Stay"

                         : " -> Goto "s + std::to_string(destination))

             << '\n';



        if(destination == localhost) {

            continue;
@@ -63,8 +68,8 @@ transmit_metadata(gkfs::rpc::host_t localhost) { 
        rpc_err_out_t out{};

        hg_addr_t host_addr{};



        in.key = metadatum.first.c_str();

        in.value = metadatum.second.c_str();

        in.key = metadentry.first.c_str();

        in.value = metadentry.second.c_str();



        auto ret = margo_addr_lookup(mid, hosts[destination].second.c_str(),

                                     &host_addr);
@@ -94,6 +99,30 @@ transmit_metadata(gkfs::rpc::host_t localhost) { 
        assert(ret == HG_SUCCESS);

    }



    // Relocate data (chunks)



    for(auto& chunks_dir : GKFS_DATA->storage()->chunks_directory_iterator()) {

        assert(chunks_dir.is_directory()); // TODO(dauer) just log

        string file_path = GKFS_DATA->storage()->get_file_path(

                chunks_dir.path().filename().string());



        for(auto& chunk_file : fs::directory_iterator(chunks_dir)) {

            assert(chunk_file.is_regular_file()); // TODO(dauer) just log

            gkfs::rpc::chnk_id_t chunk_id =

                    std::stoul(chunk_file.path().filename().string());

            auto destination = distributor.locate_data(file_path, chunk_id);

            cout << file_path << " chunk: " << chunk_id

                 << (destination == localhost

                             ? " Stay"

                             : " -> Goto "s + std::to_string(destination))

                 << '\n';



            if(destination == localhost) {

                continue;

            }

        }

    }



    margo_finalize(mid);

}