Verified Commit 7e3bbec5 authored by Marc Vef's avatar Marc Vef
Browse files

Change recording to micro seconds and storing raw bytes.

parent 125882a1

include/common/msgpack_util.hpp

+7 −7
Original line number Diff line number Diff line
@@ -51,13 +51,13 @@ public: 
    std::string hostname_;

    int pid_;



    // in milliseconds

    std::vector<double> start_t_{};

    std::vector<double> end_t_{};

    // in bytes per second

    std::vector<double> avg_{};

    // in microseconds

    std::vector<uint32_t> start_t_{};

    std::vector<uint32_t> end_t_{};

    // in bytes

    std::vector<uint32_t> req_size_{};



    uint64_t total_bytes_{};

    uint32_t total_bytes_{};

    int total_iops_{0};



    bool is_enabled_{false};
@@ -71,7 +71,7 @@ public: 
    template <class T>

    void

    pack(T& pack) {

        pack(init_t_, hostname_, pid_, start_t_, end_t_, avg_, total_iops_,

        pack(init_t_, hostname_, pid_, start_t_, end_t_, req_size_, total_iops_,

             total_bytes_);

    }

src/client/preload.cpp

+2 −0
Original line number Diff line number Diff line
@@ -315,8 +315,10 @@ destroy_preload() { 
        destroy_forwarding_mapper();

    }

#ifdef GKFS_ENABLE_CLIENT_METRICS

    LOG(INFO, "Flushing metrics...");

    CTX->write_metrics().flush_msgpack();

    CTX->read_metrics().flush_msgpack();

    LOG(INFO, "Metrics flushed.");

#endif

    CTX->clear_hosts();

    LOG(DEBUG, "Peer information deleted");

src/common/msgpack_util.cpp

+9 −16
Original line number Diff line number Diff line
@@ -55,25 +55,18 @@ ClientMetrics::add_event( 
    auto end = std::chrono::system_clock::now();



    auto start_offset =

            std::chrono::duration<double, std::milli>(start - init_t_);

    auto end_offset = std::chrono::duration<double, std::milli>(end - init_t_);

    auto duration = std::chrono::duration<double, std::milli>(end_offset -

            std::chrono::duration<double, std::micro>(start - init_t_);

    auto end_offset = std::chrono::duration<double, std::micro>(end - init_t_);

    auto duration = std::chrono::duration<double, std::micro>(end_offset -

                                                              start_offset);



    total_bytes_ += size;

    //    cout << "duration size bytes: " << size << endl;

    size /= (1024 * 1024); // in MiB

                           //    cout << "duration size MiB: " << size << endl;

    auto duration_s =

            duration.count() /

            1000; // in seconds

                  //    cout << "start: " << start_offset.count() << endl;

                  //    cout << "duration millisecond: " << duration.count() <<

                  //    endl; cout << "duration second: " << duration_s << endl;

                  //    cout << "end: " << end_offset.count() << endl;

    start_t_.emplace_back(start_offset.count());

    end_t_.emplace_back(end_offset.count());

    avg_.emplace_back((size / duration_s));

    //    auto size_mib = size / (1024 * 1024);      // in MiB

    //    auto duration_s = duration.count() / 1000; // in seconds

    // throw away decimals

    start_t_.emplace_back(static_cast<size_t>(start_offset.count()));

    end_t_.emplace_back(static_cast<size_t>(end_offset.count()));

    req_size_.emplace_back(size);

    total_iops_ += 1;

}

tools/clientmetrics2json.cpp

+28 −11
Original line number Diff line number Diff line
@@ -32,20 +32,27 @@ 


#include <iostream>

#include <fstream>

#include <filesystem>



extern "C" {

#include <unistd.h>

#include <fcntl.h>

}



using namespace std;

using json = nlohmann::json;

int

main() {



    std::ifstream file(

            "/tmp/gkfs_client_metrics/_2023-12-06_17:43:11_evie_365368.msgpack",

            std::ios::binary);

main(int argc, char** argv) {

    if(argc != 2) {

        std::cerr << "Usage: " << argv[0] << " <path_to_clientmetrics.msgpack>"

                  << std::endl;

        return -1;

    }

    auto path = std::filesystem::path(argv[1]);

    if(!std::filesystem::exists(path)) {

        std::cerr << "Input file " << path << " does not exist" << std::endl;

        return -1;

    }

    std::ifstream file(path, std::ios::binary);

    if(!file.is_open()) {

        std::cout << "failed to open " << '\n';

        return -1;
@@ -57,18 +64,28 @@ main() { 
    file.close();

    std::error_code ec{};

    auto undata = msgpack::unpack<gkfs::messagepack::ClientMetrics>(buffer, ec);



    std::vector<double> avg_thruput(undata.req_size_.size());

    for(size_t i = 0; i < avg_thruput.size(); ++i) {

        auto size_mib = undata.req_size_[i] / (1024.0 * 1024.0); // in MiB

        auto duration_s = (undata.end_t_[i] - undata.start_t_[i]) / 1000000.0;

        avg_thruput[i] = std::round((size_mib / duration_s) * 100.0) / 100.0;

    }

    json json_obj;

    json_obj["hostname"] = undata.hostname_;

    json_obj["pid"] = undata.pid_;

    json_obj["total_bytes"] = undata.total_bytes_;

    json_obj["total_iops"] = undata.total_iops_;

    json_obj["start_t"] = undata.start_t_;

    json_obj["end_t"] = undata.end_t_;

    json_obj["avg"] = undata.avg_;

    json_obj["start_t_micro"] = undata.start_t_;

    json_obj["end_t_micro"] = undata.end_t_;

    json_obj["req_size"] = undata.req_size_;

    json_obj["[extra]avg_thruput_mib"] = avg_thruput;



    std::cout << "Generated JSON:" << std::endl;

    std::cout << json_obj.dump(4) << std::endl; // Use dump for pretty printing

    for(const auto& item : json_obj.items()) {

        std::cout << item.key() << ": " << item.value().dump() << std::endl;

    }

    //    std::cout << json_obj.dump(4) << std::endl; // Use dump for pretty

    //    printing



    return 0;

}
 
 No newline at end of file