Newer
Older
/*
Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain
Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany
This software was partially supported by the
EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
This software was partially supported by the
ADA-FS project under the SPPEXA project funded by the DFG.
This file is part of GekkoFS.
GekkoFS is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
GekkoFS is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with GekkoFS. If not, see <https://www.gnu.org/licenses/>.
SPDX-License-Identifier: GPL-3.0-or-later
*/
#ifndef GKFS_COMMON_STATS_HPP
#define GKFS_COMMON_STATS_HPP
#include <cstdint>
#include <unistd.h>
#include <cassert>
#include <map>
#include <set>
#include <vector>
#include <deque>
#include <chrono>
#include <initializer_list>
#include <iomanip>
#include <fstream>
#include <prometheus/exposer.h>
#include <prometheus/registry.h>
/**
* Provides storage capabilities to provide stats about GekkoFS
*/
namespace gkfs::utils {
/*
Number of operations (Create, write/ read, remove, mkdir...)
Size of database (metadata keys, should be not needed, any)
Size of data (+write - delete)
Server Bandwidth (write / read operations)
mean, (lifetime of the server)
1 minute mean
5 minute mean
10 minute mean
we need to store the info and the timestamp to calculate it
The stats will only be calculated when requested
a cached value will be send (with a deadline)
*/
enum class IOPS_OP {
IOPS_CREATE,
IOPS_WRITE,
IOPS_READ,
IOPS_STATS,
IOPS_DIRENTS,
enum class SIZE_OP { WRITE_SIZE, READ_SIZE }; ///< enum storing Size Stats
private:
constexpr static const std::initializer_list<Stats::IOPS_OP> all_IOPS_OP = {
IOPS_OP::IOPS_CREATE,
IOPS_OP::IOPS_WRITE,
IOPS_OP::IOPS_READ,
IOPS_OP::IOPS_STATS,
IOPS_OP::IOPS_DIRENTS,
IOPS_OP::IOPS_REMOVE}; ///< Enum IOPS iterator
constexpr static const std::initializer_list<Stats::SIZE_OP> all_SIZE_OP = {
SIZE_OP::WRITE_SIZE, SIZE_OP::READ_SIZE}; ///< Enum SIZE iterator
const std::vector<std::string> IOPS_OP_S = {
"IOPS_CREATE", "IOPS_WRITE", "IOPS_READ",
"IOPS_STATS", "IOPS_DIRENTS", "IOPS_REMOVE"}; ///< Stats Labels
const std::vector<std::string> SIZE_OP_S = {"WRITE_SIZE",
"READ_SIZE"}; ///< Stats Labels
std::chrono::time_point<std::chrono::steady_clock>
start; ///< When we started the server
const unsigned int MAX_STATS = 1000000; ///< How many stats will be stored
std::map<IOPS_OP, unsigned long>
IOPS; ///< Stores total value for global mean
std::map<SIZE_OP, unsigned long>
SIZE; ///< Stores total value for global mean
std::map<IOPS_OP,
std::deque<std::chrono::time_point<std::chrono::steady_clock>>>
TIME_IOPS; ///< Stores timestamp when an operation comes removes if
///< first operation if > 10 minutes Different means will
///< be stored and cached 1 minuted
std::map<enum SIZE_OP,
std::deque<std::pair<
std::chrono::time_point<std::chrono::steady_clock>,
unsigned long long>>>
TIME_SIZE; ///< For size operations we need to store the timestamp
///< and the size
std::thread t_output; ///< Thread that outputs stats info
bool output_thread_; ///< Enables or disables the output thread
bool running =
true; ///< Controls the destruction of the class/stops the thread
/**
* @brief Sends all the stats to the screen
* Debug Function
*
* @param d is the time between output
* @param file_output is the output file
*/
void
output(std::chrono::seconds d, std::string file_output);
std::map<std::pair<std::string, unsigned long long>, unsigned int>
CHUNK_READ; ///< Stores the number of times a chunk/file is read
std::map<std::pair<std::string, unsigned long long>, unsigned int>
CHUNK_WRITE; ///< Stores the number of times a chunk/file is write
/**
* @brief Called by output to generate CHUNK map
*
* @param output is the output stream
*/
void
output_map(std::ofstream& output);
/**
* @brief Dumps all the means from the stats
* @param of Output stream
dump(std::ofstream& of);
// Prometheus Push structs
#ifdef GKFS_PROMETHEUS
std::shared_ptr<Gateway> gateway; ///< Prometheus Gateway
std::shared_ptr<Registry> registry; ///< Prometheus Counters Registry
Family<Counter>* family_counter; ///< Prometheus IOPS counter
Family<Summary>* family_summary; ///< Prometheus SIZE counter
std::map<IOPS_OP, Counter*> IOPS_Prometheus;
std::map<SIZE_OP, Summary*> SIZE_Prometheus;
public:
/**
* @brief Starts the Stats module and initializes structures
* @param output_thread creates an aditional thread that outputs the stats
* @param filename file where to write the output
Stats(bool output_thread, std::string filename,
std::string prometheus_gateway);
/**
* @brief Destroys the class, and any associated thread
*
*/
~Stats();
/**
* @brief Set the up Prometheus gateway and structures
*
* @param gateway_ip
* @param gateway_port
*/
void
setup_Prometheus(std::string gateway_ip, std::string gateway_port);
/**
* @brief Adds a new read access to the chunk/path specified
*
* @param path
* @param chunk
*/
void
add_read(std::string path, unsigned long long chunk);
/**
* @brief Adds a new write access to the chunk/path specified
*
* @param path
* @param chunk
*/
void
add_write(std::string path, unsigned long long chunk);
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
/**
* Add a new value for a IOPS, that does not involve any size
* No value needed as they are simple (1 create, 1 read...)
* Size operations internally call this operation (read,write)
*
* @param IOPS_OP Which operation to add
*/
void add_value_iops(enum IOPS_OP);
/**
* @brief Store a new stat point, with a size value.
* If it involves a IO operations it will call the corresponding
* operation
*
* @param SIZE_OP Which operation we refer
* @param value to store (SIZE_OP)
*/
void
add_value_size(enum SIZE_OP, unsigned long long value);
/**
* @brief Get the total mean value of the asked stat
* This can be provided inmediately without cost
* @return mean value
*/
double get_mean(enum IOPS_OP);
/**
* @brief Get the total mean value of the asked stat
* This can be provided inmediately without cost
* @return mean value
*/
double get_mean(enum SIZE_OP);
/**
* @brief Get all the means (total, 1,5 and 10 minutes) for a SIZE_OP
* Returns precalculated values if we just calculated them 1 minute ago
*
* @return std::vector< double > with 4 means
*/
std::vector<double> get_four_means(enum SIZE_OP);
/**
* @brief Get all the means (total, 1,5 and 10 minutes) for a IOPS_OP
* Returns precalculated values if we just calculated them 1 minute ago
*
* @return std::vector< double > with 4 means
*/
std::vector<double> get_four_means(enum IOPS_OP);