Commit deefa105 authored by Ramon Nou's avatar Ramon Nou
Browse files

Added documentation

parent c1d961c0

CHANGELOG.md

+3 −1
Original line number Diff line number Diff line
@@ -7,7 +7,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 


## [Unreleased]



- Added Stats gathering in servers

- Added Stats ([!128](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/128)) gathering in servers

- GKFS_CHUNK_STATS enables chunk usage output

- Stats output can be enabled with --output-stats <filename>

### New



- Added new experimental metadata backend:

README.md

+1 −1
Original line number Diff line number Diff line
@@ -109,7 +109,7 @@ Options: 
                              RocksDB is default if not set. Parallax support is experimental.

                              Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir.

  --parallaxsize TEXT         parallaxdb - metadata file size in GB (default 8GB), used only with new files

  --output-stats              Enables the output of the stats on the stdout (each 10s) for debug

  --output-stats TEXT         Enables the output of the stats on the FILE (each 10s) for debug

  --version                   Print version and exit.

```

docs/sphinx/users/running.md

+1 −0
Original line number Diff line number Diff line
@@ -79,6 +79,7 @@ Options: 
                              RocksDB is default if not set. Parallax support is experimental.

                              Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir.

  --parallaxsize TEXT         parallaxdb - metadata file size in GB (default 8GB), used only with new files

  --output-stats TEXT         Outputs the stats to the file each 10s.

  --version                   Print version and exit.

````

include/common/statistics/stats.hpp

+40 −35
Original line number Diff line number Diff line
@@ -76,54 +76,59 @@ public: 
        IOPS_STATS,

        IOPS_DIRENTS,

        IOPS_REMOVE,

    };

    }; ///< enum storing IOPS Stats



    enum class SIZE_OP { WRITE_SIZE, READ_SIZE };

    enum class SIZE_OP { WRITE_SIZE, READ_SIZE }; ///< enum storing Size Stats



private:

    constexpr static const std::initializer_list<Stats::IOPS_OP> all_IOPS_OP = {

            IOPS_OP::IOPS_CREATE, IOPS_OP::IOPS_WRITE,   IOPS_OP::IOPS_READ,

            IOPS_OP::IOPS_STATS,  IOPS_OP::IOPS_DIRENTS, IOPS_OP::IOPS_REMOVE};

            IOPS_OP::IOPS_CREATE,

            IOPS_OP::IOPS_WRITE,

            IOPS_OP::IOPS_READ,

            IOPS_OP::IOPS_STATS,

            IOPS_OP::IOPS_DIRENTS,

            IOPS_OP::IOPS_REMOVE}; ///< Enum IOPS iterator



    constexpr static const std::initializer_list<Stats::SIZE_OP> all_SIZE_OP = {

            SIZE_OP::WRITE_SIZE, SIZE_OP::READ_SIZE};



    const std::vector<std::string> IOPS_OP_S = {"IOPS_CREATE",  "IOPS_WRITE",

                                                "IOPS_READ",    "IOPS_STATS",

                                                "IOPS_DIRENTS", "IOPS_REMOVE"};

    const std::vector<std::string> SIZE_OP_S = {"WRITE_SIZE", "READ_SIZE"};

    std::chrono::time_point<std::chrono::steady_clock> last_cached;

    /* Measures when we started the server */

    std::chrono::time_point<std::chrono::steady_clock> start;

    // How many stats will be stored

    const unsigned int MAX_STATS = 1000000;



    // Stores total value for global mean

    std::map<IOPS_OP, unsigned long> IOPS;

    std::map<SIZE_OP, unsigned long> SIZE;





    // Stores timestamp when an operation comes

    // removes if first operation if > 10 minutes

    // Different means will be stored and cached 1 minuted

            SIZE_OP::WRITE_SIZE, SIZE_OP::READ_SIZE}; ///< Enum SIZE iterator



    const std::vector<std::string> IOPS_OP_S = {

            "IOPS_CREATE", "IOPS_WRITE",   "IOPS_READ",

            "IOPS_STATS",  "IOPS_DIRENTS", "IOPS_REMOVE"}; ///< Stats Labels

    const std::vector<std::string> SIZE_OP_S = {"WRITE_SIZE",

                                                "READ_SIZE"}; ///< Stats Labels



    std::chrono::time_point<std::chrono::steady_clock>

            start; ///< When we started the server



    const unsigned int MAX_STATS = 1000000; ///< How many stats will be stored





    std::map<IOPS_OP, unsigned long>

            IOPS; ///< Stores total value for global mean

    std::map<SIZE_OP, unsigned long>

            SIZE; ///< Stores total value for global mean



    std::map<IOPS_OP,

             std::deque<std::chrono::time_point<std::chrono::steady_clock>>>

            TIME_IOPS;

            TIME_IOPS; ///< Stores timestamp when an operation comes removes if

                       ///< first operation if > 10 minutes Different means will

                       ///< be stored and cached 1 minuted





    // For size operations we need to store the timestamp and

    // the size

    std::map<enum SIZE_OP,

             std::deque<std::pair<

                     std::chrono::time_point<std::chrono::steady_clock>,

                     unsigned long long>>>

            TIME_SIZE;

            TIME_SIZE; ///< For size operations we need to store the timestamp

                       ///< and the size





    // Thread that outputs stats info

    std::thread t_output;

    bool output_thread_;

    std::thread t_output; ///< Thread that outputs stats info

    bool output_thread_;  ///< Enables or disables the output thread



    // Controls the destruction of the class/stops the thread

    bool running = true;

    bool running =

            true; ///< Controls the destruction of the class/stops the thread

    /**

     * @brief Sends all the stats to the screen

     * Debug Function
@@ -135,9 +140,9 @@ private: 
    output(std::chrono::seconds d, std::string file_output);



    std::map<std::pair<std::string, unsigned long long>, unsigned int>

            CHUNK_READ;

            CHUNK_READ; ///< Stores the number of times a chunk/file is read

    std::map<std::pair<std::string, unsigned long long>, unsigned int>

            CHUNK_WRITE;

            CHUNK_WRITE; ///< Stores the number of times a chunk/file is write



    /**

     * @brief Called by output to generate CHUNK map