Commit 7946ab25 authored by Marc Vef's avatar Marc Vef
Browse files

Chunk_util into gkfs::util namespace + comment added

parent f6ba5693

include/global/chunk_calc_util.hpp

+15 −7
Original line number Diff line number Diff line
@@ -16,6 +16,9 @@ 


#include <cassert>



namespace gkfs {

namespace util {



/**

 * Compute the base2 logarithm for 64 bit integers

 */
@@ -28,8 +31,6 @@ inline int log2(uint64_t n) { 
            57, 46, 52, 38, 26, 32, 41, 50, 36, 17, 19, 29, 10, 13, 21, 56,

            45, 25, 31, 35, 16, 9, 12, 44, 24, 15, 8, 23, 7, 6, 5, 63 };



    assert(n > 0); // TODO This needs to be removed and a check for CHUNKSIZE has to be put somewhere



    n |= n >> 1;

    n |= n >> 2;

    n |= n >> 4;
@@ -88,7 +89,11 @@ inline size_t chnk_rpad(const off64_t offset, const size_t chnk_size) { 
 * chunk_id(0,4) = 0;

 */

inline uint64_t chnk_id_for_offset(const off64_t offset, const size_t chnk_size) {

    // TODO This does not work for very large offsets: `offset / chnk_size` works

    /*

     * This does not work for offsets that use the 64th bit, i.e., 9223372036854775808.

     * 9223372036854775808 - 1 uses 63 bits and still works. `offset / chnk_size` works with the 64th bit.

     * With this number we can address more than 19,300,000 exabytes of data though. Hi future me?

     */

    return static_cast<uint64_t>(chnk_lalign(offset, chnk_size) >> log2(chnk_size));

}
@@ -106,4 +111,7 @@ inline uint64_t chnk_count_for_offset(const off64_t offset, const size_t count, 
                                 (chnk_start >> log2(chnk_size)) + 1);

}



} // namespace util

} // namespace gkfs



#endif

src/client/rpc/forward_data.cpp

+13 −12
Original line number Diff line number Diff line
@@ -42,8 +42,8 @@ ssize_t forward_write(const string& path, const void* buf, const bool append_fla 
    // which interval to look for chunks

    off64_t offset = append_flag ? in_offset : (updated_metadentry_size - write_size);



    auto chnk_start = chnk_id_for_offset(offset, gkfs::config::rpc::chunksize);

    auto chnk_end = chnk_id_for_offset((offset + write_size) - 1, gkfs::config::rpc::chunksize);

    auto chnk_start = gkfs::util::chnk_id_for_offset(offset, gkfs::config::rpc::chunksize);

    auto chnk_end = gkfs::util::chnk_id_for_offset((offset + write_size) - 1, gkfs::config::rpc::chunksize);



    // Collect all chunk ids within count that have the same destination so

    // that those are send in one rpc bulk transfer
@@ -108,12 +108,12 @@ ssize_t forward_write(const string& path, const void* buf, const bool append_fla 


        // receiver of first chunk must subtract the offset from first chunk

        if (target == chnk_start_target) {

            total_chunk_size -= chnk_lpad(offset, gkfs::config::rpc::chunksize);

            total_chunk_size -= gkfs::util::chnk_lpad(offset, gkfs::config::rpc::chunksize);

        }



        // receiver of last chunk must subtract

        if (target == chnk_end_target) {

            total_chunk_size -= chnk_rpad(offset + write_size, gkfs::config::rpc::chunksize);

            total_chunk_size -= gkfs::util::chnk_rpad(offset + write_size, gkfs::config::rpc::chunksize);

        }



        auto endp = CTX->hosts().at(target);
@@ -126,7 +126,7 @@ ssize_t forward_write(const string& path, const void* buf, const bool append_fla 
                    path,

                    // first offset in targets is the chunk with

                    // a potential offset

                    chnk_lpad(offset, gkfs::config::rpc::chunksize),

                    gkfs::util::chnk_lpad(offset, gkfs::config::rpc::chunksize),

                    target,

                    CTX->hosts().size(),

                    // number of chunks handled by that destination
@@ -198,8 +198,8 @@ ssize_t forward_read(const string& path, void* buf, const off64_t offset, const 


    // Calculate chunkid boundaries and numbers so that daemons know in which

    // interval to look for chunks

    auto chnk_start = chnk_id_for_offset(offset, gkfs::config::rpc::chunksize);

    auto chnk_end = chnk_id_for_offset((offset + read_size - 1), gkfs::config::rpc::chunksize);

    auto chnk_start = gkfs::util::chnk_id_for_offset(offset, gkfs::config::rpc::chunksize);

    auto chnk_end = gkfs::util::chnk_id_for_offset((offset + read_size - 1), gkfs::config::rpc::chunksize);



    // Collect all chunk ids within count that have the same destination so

    // that those are send in one rpc bulk transfer
@@ -264,12 +264,12 @@ ssize_t forward_read(const string& path, void* buf, const off64_t offset, const 


        // receiver of first chunk must subtract the offset from first chunk

        if (target == chnk_start_target) {

            total_chunk_size -= chnk_lpad(offset, gkfs::config::rpc::chunksize);

            total_chunk_size -= gkfs::util::chnk_lpad(offset, gkfs::config::rpc::chunksize);

        }



        // receiver of last chunk must subtract

        if (target == chnk_end_target) {

            total_chunk_size -= chnk_rpad(offset + read_size, gkfs::config::rpc::chunksize);

            total_chunk_size -= gkfs::util::chnk_rpad(offset + read_size, gkfs::config::rpc::chunksize);

        }



        auto endp = CTX->hosts().at(target);
@@ -282,7 +282,7 @@ ssize_t forward_read(const string& path, void* buf, const off64_t offset, const 
                    path,

                    // first offset in targets is the chunk with

                    // a potential offset

                    chnk_lpad(offset, gkfs::config::rpc::chunksize),

                    gkfs::util::chnk_lpad(offset, gkfs::config::rpc::chunksize),

                    target,

                    CTX->hosts().size(),

                    // number of chunks handled by that destination
@@ -355,8 +355,9 @@ int forward_truncate(const std::string& path, size_t current_size, size_t new_si 


    // Find out which data servers need to delete data chunks in order to

    // contact only them

    const unsigned int chunk_start = chnk_id_for_offset(new_size, gkfs::config::rpc::chunksize);

    const unsigned int chunk_end = chnk_id_for_offset(current_size - new_size - 1, gkfs::config::rpc::chunksize);

    const unsigned int chunk_start = gkfs::util::chnk_id_for_offset(new_size, gkfs::config::rpc::chunksize);

    const unsigned int chunk_end = gkfs::util::chnk_id_for_offset(current_size - new_size - 1,

                                                                  gkfs::config::rpc::chunksize);



    std::unordered_set<unsigned int> hosts;

    for (unsigned int chunk_id = chunk_start; chunk_id <= chunk_end; ++chunk_id) {

src/daemon/handler/srv_data.cpp

+2 −2
Original line number Diff line number Diff line
@@ -524,10 +524,10 @@ static hg_return_t rpc_srv_truncate(hg_handle_t handle) { 
    }

    GKFS_DATA->spdlogger()->debug("{}() path: '{}', length: {}", __func__, in.path, in.length);



    unsigned int chunk_start = chnk_id_for_offset(in.length, gkfs::config::rpc::chunksize);

    unsigned int chunk_start = gkfs::util::chnk_id_for_offset(in.length, gkfs::config::rpc::chunksize);



    // If we trunc in the the middle of a chunk, do not delete that chunk

    auto left_pad = chnk_lpad(in.length, gkfs::config::rpc::chunksize);

    auto left_pad = gkfs::util::chnk_lpad(in.length, gkfs::config::rpc::chunksize);

    if (left_pad != 0) {

        GKFS_DATA->storage()->truncate_chunk(in.path, chunk_start, left_pad);

        ++chunk_start;