LCOV - code coverage report
Current view: top level - include/common/statistics - stats.hpp (source / functions) Hit Total Coverage
Test: coverage.info Lines: 2 2 100.0 %
Date: 2024-04-23 00:09:24 Functions: 0 0 -
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*
       2             :   Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain
       3             :   Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany
       4             : 
       5             :   This software was partially supported by the
       6             :   EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
       7             : 
       8             :   This software was partially supported by the
       9             :   ADA-FS project under the SPPEXA project funded by the DFG.
      10             : 
      11             :   This file is part of GekkoFS.
      12             : 
      13             :   GekkoFS is free software: you can redistribute it and/or modify
      14             :   it under the terms of the GNU General Public License as published by
      15             :   the Free Software Foundation, either version 3 of the License, or
      16             :   (at your option) any later version.
      17             : 
      18             :   GekkoFS is distributed in the hope that it will be useful,
      19             :   but WITHOUT ANY WARRANTY; without even the implied warranty of
      20             :   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      21             :   GNU General Public License for more details.
      22             : 
      23             :   You should have received a copy of the GNU General Public License
      24             :   along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.
      25             : 
      26             :   SPDX-License-Identifier: GPL-3.0-or-later
      27             : */
      28             : 
      29             : #ifndef GKFS_COMMON_STATS_HPP
      30             : #define GKFS_COMMON_STATS_HPP
      31             : 
      32             : #include <cstdint>
      33             : #include <unistd.h>
      34             : #include <cassert>
      35             : #include <map>
      36             : #include <set>
      37             : #include <vector>
      38             : #include <deque>
      39             : #include <chrono>
      40             : #include <optional>
      41             : #include <initializer_list>
      42             : #include <thread>
      43             : #include <iostream>
      44             : #include <iomanip>
      45             : #include <fstream>
      46             : #include <atomic>
      47             : #include <mutex>
      48             : #include <config.hpp>
      49             : 
      50             : 
      51             : // PROMETHEUS includes
      52             : #ifdef GKFS_ENABLE_PROMETHEUS
      53             : #include <prometheus/counter.h>
      54             : #include <prometheus/summary.h>
      55             : #include <prometheus/exposer.h>
      56             : #include <prometheus/registry.h>
      57             : #include <prometheus/gateway.h>
      58             : 
      59             : using namespace prometheus;
      60             : #endif
      61             : 
      62             : 
      63             : /**
      64             :  * Provides storage capabilities to provide stats about GekkoFS
      65             :  * The information is per server.
      66             :  * We do not provide accurate stats for 1-5-10 minute stats
      67             :  *
      68             :  */
      69             : namespace gkfs::utils {
      70             : 
      71             : /**
      72             :  *
      73             :  * Number of operations (Create, write/ read, remove, mkdir...)
      74             :  * Size of database (metadata keys, should be not needed, any)
      75             :  * Size of data (+write - delete)
      76             :  * Server Bandwidth (write / read operations)
      77             :  *
      78             :  * mean, (lifetime of the server)
      79             :  * 1 minute mean
      80             :  * 5 minute mean
      81             :  * 10 minute mean
      82             :  *
      83             :  * To provide the stats that we need,
      84             :  * we need to store the info and the timestamp to calculate it
      85             :  * A vector should work, with a maximum of elements,
      86             :  */
      87             : 
      88             : class Stats {
      89             : public:
      90             :     enum class IopsOp {
      91             :         iops_create,
      92             :         iops_write,
      93             :         iops_read,
      94             :         iops_stats,
      95             :         iops_dirent,
      96             :         iops_remove,
      97             :     }; ///< enum storing IOPS Stats
      98             : 
      99             :     enum class SizeOp { write_size, read_size }; ///< enum storing Size Stats
     100             : 
     101             : private:
     102             :     constexpr static const std::initializer_list<Stats::IopsOp> all_IopsOp = {
     103             :             IopsOp::iops_create, IopsOp::iops_write,
     104             :             IopsOp::iops_read,   IopsOp::iops_stats,
     105             :             IopsOp::iops_dirent, IopsOp::iops_remove}; ///< Enum IOPS iterator
     106             : 
     107             :     constexpr static const std::initializer_list<Stats::SizeOp> all_SizeOp = {
     108             :             SizeOp::write_size, SizeOp::read_size}; ///< Enum SIZE iterator
     109             : 
     110             :     const std::vector<std::string> IopsOp_s = {
     111             :             "IOPS_CREATE", "IOPS_WRITE",   "IOPS_READ",
     112          66 :             "IOPS_STATS",  "IOPS_DIRENTS", "IOPS_REMOVE"}; ///< Stats Labels
     113             :     const std::vector<std::string> SizeOp_s = {"WRITE_SIZE",
     114          66 :                                                "READ_SIZE"}; ///< Stats Labels
     115             : 
     116             :     std::chrono::time_point<std::chrono::steady_clock>
     117             :             start; ///< When we started the server
     118             : 
     119             : 
     120             :     std::map<IopsOp, std::atomic<unsigned long>>
     121             :             iops_mean; ///< Stores total value for global mean
     122             :     std::map<SizeOp, std::atomic<unsigned long>>
     123             :             size_mean; ///< Stores total value for global mean
     124             : 
     125             :     std::mutex time_iops_mutex;
     126             :     std::mutex size_iops_mutex;
     127             : 
     128             :     std::map<IopsOp,
     129             :              std::deque<std::chrono::time_point<std::chrono::steady_clock>>>
     130             :             time_iops; ///< Stores timestamp when an operation comes removes if
     131             :                        ///< first operation if > 10 minutes Different means will
     132             :                        ///< be stored and cached 1 minuted
     133             : 
     134             : 
     135             :     std::map<SizeOp, std::deque<std::pair<
     136             :                              std::chrono::time_point<std::chrono::steady_clock>,
     137             :                              unsigned long long>>>
     138             :             time_size; ///< For size operations we need to store the timestamp
     139             :                        ///< and the size
     140             : 
     141             : 
     142             :     std::thread t_output;    ///< Thread that outputs stats info
     143             :     bool output_thread_;     ///< Enables or disables the output thread
     144             :     bool enable_prometheus_; ///< Enables or disables the prometheus output
     145             :     bool enable_chunkstats_; ///< Enables or disables the chunk stats output
     146             : 
     147             : 
     148             :     bool running =
     149             :             true; ///< Controls the destruction of the class/stops the thread
     150             :     /**
     151             :      * @brief Sends all the stats to the screen
     152             :      * Debug Function
     153             :      *
     154             :      * @param d is the time between output
     155             :      * @param file_output is the output file
     156             :      */
     157             :     void
     158             :     output(std::chrono::seconds d, std::string file_output);
     159             : 
     160             :     std::map<std::pair<std::string, unsigned long long>,
     161             :              std::atomic<unsigned int>>
     162             :             chunk_reads; ///< Stores the number of times a chunk/file is read
     163             :     std::map<std::pair<std::string, unsigned long long>,
     164             :              std::atomic<unsigned int>>
     165             :             chunk_writes; ///< Stores the number of times a chunk/file is write
     166             : 
     167             :     /**
     168             :      * @brief Called by output to generate CHUNK map
     169             :      *
     170             :      * @param output is the output stream
     171             :      */
     172             :     void
     173             :     output_map(std::ofstream& output);
     174             : 
     175             : 
     176             :     /**
     177             :      * @brief Dumps all the means from the stats
     178             :      * @param of Output stream
     179             :      */
     180             :     void
     181             :     dump(std::ofstream& of);
     182             : 
     183             : 
     184             : // Prometheus Push structs
     185             : #ifdef GKFS_ENABLE_PROMETHEUS
     186             :     std::shared_ptr<Gateway> gateway;   ///< Prometheus Gateway
     187             :     std::shared_ptr<Registry> registry; ///< Prometheus Counters Registry
     188             :     Family<Counter>* family_counter;    ///< Prometheus IOPS counter (managed by
     189             :                                         ///< Prometheus cpp)
     190             :     Family<Summary>* family_summary;    ///< Prometheus SIZE counter (managed by
     191             :                                         ///< Prometheus cpp)
     192             :     std::map<IopsOp, Counter*> iops_prometheus; ///< Prometheus IOPS metrics
     193             :     std::map<SizeOp, Summary*> size_prometheus; ///< Prometheus SIZE metrics
     194             : #endif
     195             : 
     196             : public:
     197             :     /**
     198             :      * @brief Starts the Stats module and initializes structures
     199             :      * @param enable_chunkstats Enables or disables the chunk stats
     200             :      * @param enable_prometheus Enables or disables the prometheus output
     201             :      * @param filename file where to write the output
     202             :      * @param prometheus_gateway ip:port to expose the metrics
     203             :      */
     204             :     Stats(bool enable_chunkstats, bool enable_prometheus,
     205             :           const std::string& filename, const std::string& prometheus_gateway);
     206             : 
     207             :     /**
     208             :      * @brief Destroys the class, and any associated thread
     209             :      *
     210             :      */
     211             :     ~Stats();
     212             : 
     213             : 
     214             :     /**
     215             :      * @brief Set the up Prometheus gateway and structures
     216             :      *
     217             :      * @param gateway_ip ip of the prometheus gateway
     218             :      * @param gateway_port port of the prometheus gateway
     219             :      */
     220             :     void
     221             :     setup_Prometheus(const std::string& gateway_ip,
     222             :                      const std::string& gateway_port);
     223             : 
     224             :     /**
     225             :      * @brief Adds a new read access to the chunk/path specified
     226             :      *
     227             :      * @param path path of the chunk
     228             :      * @param chunk chunk number
     229             :      */
     230             :     void
     231             :     add_read(const std::string& path, unsigned long long chunk);
     232             :     /**
     233             :      * @brief Adds a new write access to the chunk/path specified
     234             :      *
     235             :      * @param path path of the chunk
     236             :      * @param chunk chunk number
     237             :      */
     238             :     void
     239             :     add_write(const std::string& path, unsigned long long chunk);
     240             : 
     241             : 
     242             :     /**
     243             :      * Add a new value for a IOPS, that does not involve any size
     244             :      * No value needed as they are simple (1 create, 1 read...)
     245             :      * Size operations internally call this operation (read,write)
     246             :      *
     247             :      * @param IopsOp Which operation to add
     248             :      */
     249             : 
     250             :     void add_value_iops(enum IopsOp);
     251             : 
     252             :     /**
     253             :      * @brief Store a new stat point, with a size value.
     254             :      * If it involves a IO operations it will call the corresponding
     255             :      * operation
     256             :      *
     257             :      * @param SizeOp Which operation we refer
     258             :      * @param value to store (SizeOp)
     259             :      */
     260             :     void
     261             :     add_value_size(enum SizeOp, unsigned long long value);
     262             : 
     263             :     /**
     264             :      * @brief Get the total mean value of the asked stat
     265             :      * This can be provided inmediately without cost
     266             :      * @param IopsOp Which operation to get
     267             :      * @return mean value
     268             :      */
     269             :     double get_mean(enum IopsOp);
     270             : 
     271             : 
     272             :     /**
     273             :      * @brief Get the total mean value of the asked stat
     274             :      * This can be provided inmediately without cost
     275             :      * @param SizeOp Which operation to get
     276             :      * @return mean value
     277             :      */
     278             :     double get_mean(enum SizeOp);
     279             : 
     280             :     /**
     281             :      * @brief Get all the means (total, 1,5 and 10 minutes) for a SIZE_OP
     282             :      * Returns precalculated values if we just calculated them 1 minute ago
     283             :      * @param SizeOp Which operation to get
     284             :      *
     285             :      * @return std::vector< double > with 4 means
     286             :      */
     287             :     std::vector<double> get_four_means(enum SizeOp);
     288             : 
     289             :     /**
     290             :      * @brief Get all the means (total, 1,5 and 10 minutes) for a IOPS_OP
     291             :      * Returns precalculated values if we just calculated them 1 minute ago
     292             :      * @param IopsOp Which operation to get
     293             :      *
     294             :      * @return std::vector< double > with 4 means
     295             :      */
     296             :     std::vector<double> get_four_means(enum IopsOp);
     297             : };
     298             : 
     299             : } // namespace gkfs::utils
     300             : 
     301             : #endif // GKFS_COMMON_STATS_HPP

Generated by: LCOV version 1.16