Commit 86599ac4 authored by Ramon Nou's avatar Ramon Nou
Browse files

updated CMake 

Sampling output Thread

First stats test

Added Stats argument to enable output
parent 4fb3a7b7

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 


## [Unreleased]



- Added Stats gathering in servers

### New



- Added new experimental metadata backend:

README.md

+1 −0
Original line number Diff line number Diff line
@@ -109,6 +109,7 @@ Options: 
                              RocksDB is default if not set. Parallax support is experimental.

                              Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir.

  --parallaxsize TEXT         parallaxdb - metadata file size in GB (default 8GB), used only with new files

  --output-stats              Enables the output of the stats on the stdout (each 10s) for debug

  --version                   Print version and exit.

```

include/common/statistics/stats.hpp

+125 −85
Original line number Diff line number Diff line
@@ -37,6 +37,8 @@ 
#include <deque>

#include <chrono>

#include <initializer_list>

#include <thread>

#include <iostream>

/**

 * Provides storage capabilities to provide stats about GekkoFS

 * The information is per server.
@@ -63,6 +65,7 @@ namespace gkfs::utils { 
    a cached value will be send (with a deadline)

    */

class Stats {

public:

    enum class IOPS_OP {

        IOPS_CREATE,

        IOPS_WRITE,
@@ -72,17 +75,22 @@ class Stats{ 
        IOPS_REMOVE,

    };



    constexpr static const std::initializer_list<Stats::IOPS_OP> all_IOPS_OP = {IOPS_OP::IOPS_CREATE, IOPS_OP::IOPS_WRITE, IOPS_OP::IOPS_READ, IOPS_OP::IOPS_MKDIR,IOPS_OP::IOPS_RMDIR, IOPS_OP::IOPS_REMOVE};

    enum class SIZE_OP { METADATA_SIZE, WRITE_SIZE, READ_SIZE, DATA_SIZE };



    enum class SIZE_OP {

        METADATA_SIZE,

        WRITE_SIZE,

        READ_SIZE,

        DATA_SIZE      

    };

private:

    constexpr static const std::initializer_list<Stats::IOPS_OP> all_IOPS_OP = {

            IOPS_OP::IOPS_CREATE, IOPS_OP::IOPS_WRITE, IOPS_OP::IOPS_READ,

            IOPS_OP::IOPS_MKDIR,  IOPS_OP::IOPS_RMDIR, IOPS_OP::IOPS_REMOVE};



    constexpr static const std::initializer_list<Stats::SIZE_OP> all_SIZE_OP = {SIZE_OP::METADATA_SIZE, SIZE_OP::DATA_SIZE, SIZE_OP::WRITE_SIZE, SIZE_OP::READ_SIZE};

    constexpr static const std::initializer_list<Stats::SIZE_OP> all_SIZE_OP = {

            SIZE_OP::METADATA_SIZE, SIZE_OP::DATA_SIZE, SIZE_OP::WRITE_SIZE,

            SIZE_OP::READ_SIZE};



    const std::vector<std::string> IOPS_OP_S = {"IOPS_CREATE", "IOPS_WRITE",

                                                "IOPS_READ",   "IOPS_MKDIR",

                                                "IOPS_RMDIR",  "IOPS_REMOVE"};

    const std::vector<std::string> SIZE_OP_S = {"METADATA_SIZE", "WRITE_SIZE",

                                                "READ_SIZE", "DATA_SIZE"};

    std::chrono::time_point<std::chrono::steady_clock> last_cached;

    /* Measures when we started the server */

    std::chrono::time_point<std::chrono::steady_clock> start;
@@ -97,26 +105,49 @@ class Stats{ 
    // Stores timestamp when an operation comes

    // removes if first operation if > 10 minutes

    // Different means will be stored and cached 1 minuted

    std::map <IOPS_OP, std::deque<  std::chrono::time_point<std::chrono::steady_clock> > > TIME_IOPS;

    std::map<IOPS_OP,

             std::deque<std::chrono::time_point<std::chrono::steady_clock>>>

            TIME_IOPS;

    // We will store 1, 5, and 10 minute mean;

    std::map<IOPS_OP, std::vector<double>> CACHED_IOPS;



    // For size operations we need to store the timestamp and

    // the size

    std::map<enum SIZE_OP,

                std::deque < 

                    std::pair <  std::chrono::time_point<std::chrono::steady_clock> , unsigned long long > >

             > TIME_SIZE;

             std::deque<std::pair<

                     std::chrono::time_point<std::chrono::steady_clock>,

                     unsigned long long>>>

            TIME_SIZE;

    // We will store 1, 5, and 10 minute mean;

    std::map<enum SIZE_OP, std::vector<double>> CACHED_SIZE;



    // Thread that outputs stats info

    std::thread t_output;

    bool output_thread_;



    // Controls the destruction of the class/stops the thread

    bool running = true;

    /**

 * @brief Starts the Stats module and initializes structures

     * @brief Sends all the stats to the screen

     * Debug Function

     *

     * @param d is the time between output

     */

    void

    output(std::chrono::seconds d);



public:

    Stats();

    /**

     * @brief Starts the Stats module and initializes structures

     *

     */

    Stats(bool output_thread);



    /**

     * @brief Destroys the class, and any associated thread

     *

     */

    ~Stats();



    /**

     * Add a new value for a IOPS, that does not involve any size
@@ -136,7 +167,8 @@ void add_value_iops (enum IOPS_OP); 
     * @param SIZE_OP Which operation we refer

     * @param value to store (SIZE_OP)

     */

void add_value_size (enum SIZE_OP, unsigned long long value);

    void

    add_value_size(enum SIZE_OP, unsigned long long value);



    /**

     * @brief Get the total mean value of the asked stat
@@ -168,6 +200,14 @@ std::vector< double > get_four_means (enum SIZE_OP); 
     * @return std::vector< double > with 4 means

     */

    std::vector<double> get_four_means(enum IOPS_OP);





    /**

     * @brief Dumps all the means from the stats

     *

     */

    void

    dump();

};



} // namespace gkfs::utils

include/daemon/classes/fs_data.hpp

+10 −0
Original line number Diff line number Diff line
@@ -92,6 +92,7 @@ private: 


    // Statistics

    std::shared_ptr<gkfs::utils::Stats> stats_;

    bool output_stats_ = false;



public:

    static FsData*
@@ -224,8 +225,17 @@ public: 
    void

    stats(const std::shared_ptr<gkfs::utils::Stats>& stats);



    void

    close_stats();



    bool

    output_stats() const;



    void

    output_stats(bool output_stats);

};





} // namespace daemon

} // namespace gkfs

src/common/CMakeLists.txt

+9 −0
Original line number Diff line number Diff line
@@ -39,6 +39,15 @@ target_sources(distributor 
    ${CMAKE_CURRENT_LIST_DIR}/rpc/distributor.cpp

    )



add_library(statistics STATIC)

set_property(TARGET statistics PROPERTY POSITION_INDEPENDENT_CODE ON)

target_sources(statistics

    PUBLIC

    ${INCLUDE_DIR}/common/statistics/stats.hpp

    PRIVATE

    ${CMAKE_CURRENT_LIST_DIR}/statistics/stats.cpp

    )



if(GKFS_ENABLE_CODE_COVERAGE)

  target_code_coverage(distributor AUTO)

endif()