Verified Commit 2379b95b authored by Marc Vef's avatar Marc Vef
Browse files

dentry buffer size now per host and not total

parent 5606451a

include/config.hpp

+5 −4
Original line number Diff line number Diff line
@@ -44,7 +44,7 @@ constexpr auto daemon_log_level = 4; // info 
namespace metadata {

// which metadata should be considered apart from size and mode

constexpr auto use_atime = false;

constexpr auto use_ctime = false;

constexpr auto use_ctime = true;

constexpr auto use_mtime = false;

constexpr auto use_link_cnt = false;

constexpr auto use_blocks = false;
@@ -65,15 +65,16 @@ constexpr auto create_exist_check = true; 
namespace rpc {

constexpr auto chunksize = 524288; // in bytes (e.g., 524288 == 512KB)

// size of preallocated buffer to hold directory entries in rpc call

constexpr auto dirents_buff_size = (8 * 1024 * 1024); // 8 mega

// constexpr auto dirents_buff_size = (2147483647); // 8 mega

constexpr auto dirents_buff_size = (32 * 1024 * 1024); // 8 mega

/*

 * Indicates the number of concurrent progress to drive I/O operations of chunk

 * files to and from local file systems The value is directly mapped to created

 * Argobots xstreams, controlled in a single pool with ABT_snoozer scheduler

 */

constexpr auto daemon_io_xstreams = 8;

constexpr auto daemon_io_xstreams = 4;

// Number of threads used for RPC handlers at the daemon

constexpr auto daemon_handler_xstreams = 8;

constexpr auto daemon_handler_xstreams = 4;

} // namespace rpc



namespace rocksdb {

src/client/rpc/forward_metadata.cpp

+20 −9
Original line number Diff line number Diff line
@@ -419,12 +419,18 @@ forward_get_dirents(const string& path) { 
     * It turns out that this operation is increadibly slow for such a big

     * buffer. Moreover we don't need a zeroed buffer here.

     */

    auto large_buffer = std::unique_ptr<char[]>(

            new char[gkfs::config::rpc::dirents_buff_size]);



    // auto large_buffer = std::unique_ptr<char[]>(new

    // char[gkfs::config::rpc::dirents_buff_size]);

    // XXX there is a rounding error here depending on the number of targets...

    const std::size_t per_host_buff_size =

            gkfs::config::rpc::dirents_buff_size / targets.size();

    // const std::size_t per_host_buff_size =

    // (gkfs::config::rpc::dirents_buff_size) / targets.size();



    std::vector<std::unique_ptr<char[]>> large_buffers;

    for(std::size_t i = 0; i < targets.size(); i++) {

        large_buffers.emplace_back(std::unique_ptr<char[]>(

                new char[gkfs::config::rpc::dirents_buff_size]));

    }

    const std::size_t per_host_buff_size = gkfs::config::rpc::dirents_buff_size;



    // expose local buffers for RMA from servers

    std::vector<hermes::exposed_memory> exposed_buffers;
@@ -433,9 +439,14 @@ forward_get_dirents(const string& path) { 
    for(std::size_t i = 0; i < targets.size(); ++i) {

        try {

            exposed_buffers.emplace_back(ld_network_service->expose(

                    std::vector<hermes::mutable_buffer>{hermes::mutable_buffer{

                            large_buffer.get() + (i * per_host_buff_size),

                            per_host_buff_size}},

                    std::vector<hermes::mutable_buffer>{

                            hermes::mutable_buffer{large_buffers[i].get(),

                                                   per_host_buff_size}

                            // hermes::mutable_buffer{

                            //        large_buffer.get() + (i *

                            //        per_host_buff_size), per_host_buff_size

                            //}

                    },

                    hermes::access_mode::write_only));

        } catch(const std::exception& ex) {

            LOG(ERROR, "{}() Failed to expose buffers for RMA. err '{}'",