Program Listing for File stats.hpp

↰ Return to documentation for file (include/common/statistics/stats.hpp)

/*
  Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain
  Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany

  This software was partially supported by the
  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).

  This software was partially supported by the
  ADA-FS project under the SPPEXA project funded by the DFG.

  This file is part of GekkoFS.

  GekkoFS is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 3 of the License, or
  (at your option) any later version.

  GekkoFS is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.

  SPDX-License-Identifier: GPL-3.0-or-later
*/

#ifndef GKFS_COMMON_STATS_HPP
#define GKFS_COMMON_STATS_HPP

#include <cstdint>
#include <unistd.h>
#include <cassert>
#include <map>
#include <set>
#include <vector>
#include <deque>
#include <chrono>
#include <optional>
#include <initializer_list>
#include <thread>
#include <iostream>
#include <iomanip>
#include <fstream>
#include <atomic>
#include <mutex>
#include <config.hpp>


// PROMETHEUS includes
#ifdef GKFS_ENABLE_PROMETHEUS
#include <prometheus/counter.h>
#include <prometheus/summary.h>
#include <prometheus/exposer.h>
#include <prometheus/registry.h>
#include <prometheus/gateway.h>

using namespace prometheus;
#endif


namespace gkfs::utils {

class Stats {
public:
    enum class IopsOp {
        iops_create,
        iops_write,
        iops_read,
        iops_stats,
        iops_dirent,
        iops_remove,
    };

    enum class SizeOp { write_size, read_size };

private:
    constexpr static const std::initializer_list<Stats::IopsOp> all_IopsOp = {
            IopsOp::iops_create, IopsOp::iops_write,
            IopsOp::iops_read,   IopsOp::iops_stats,
            IopsOp::iops_dirent, IopsOp::iops_remove};

    constexpr static const std::initializer_list<Stats::SizeOp> all_SizeOp = {
            SizeOp::write_size, SizeOp::read_size};

    const std::vector<std::string> IopsOp_s = {
            "IOPS_CREATE", "IOPS_WRITE",   "IOPS_READ",
            "IOPS_STATS",  "IOPS_DIRENTS", "IOPS_REMOVE"};
    const std::vector<std::string> SizeOp_s = {"WRITE_SIZE",
                                               "READ_SIZE"};

    std::chrono::time_point<std::chrono::steady_clock>
            start;


    std::map<IopsOp, std::atomic<unsigned long>>
            iops_mean;
    std::map<SizeOp, std::atomic<unsigned long>>
            size_mean;

    std::mutex time_iops_mutex;
    std::mutex size_iops_mutex;

    std::map<IopsOp,
             std::deque<std::chrono::time_point<std::chrono::steady_clock>>>
            time_iops;


    std::map<SizeOp, std::deque<std::pair<
                             std::chrono::time_point<std::chrono::steady_clock>,
                             unsigned long long>>>
            time_size;


    std::thread t_output;
    bool output_thread_;
    bool enable_prometheus_;
    bool enable_chunkstats_;


    bool running =
            true;
    void
    output(std::chrono::seconds d, std::string file_output);

    std::map<std::pair<std::string, unsigned long long>,
             std::atomic<unsigned int>>
            chunk_reads;
    std::map<std::pair<std::string, unsigned long long>,
             std::atomic<unsigned int>>
            chunk_writes;

    void
    output_map(std::ofstream& output);


    void
    dump(std::ofstream& of);


// Prometheus Push structs
#ifdef GKFS_ENABLE_PROMETHEUS
    std::shared_ptr<Gateway> gateway;
    std::shared_ptr<Registry> registry;
    Family<Counter>* family_counter;
    Family<Summary>* family_summary;
    std::map<IopsOp, Counter*> iops_prometheus;
    std::map<SizeOp, Summary*> size_prometheus;
#endif

public:
    Stats(bool enable_chunkstats, bool enable_prometheus,
          const std::string& filename, const std::string& prometheus_gateway);

    ~Stats();


    void
    setup_Prometheus(const std::string& gateway_ip,
                     const std::string& gateway_port);

    void
    add_read(const std::string& path, unsigned long long chunk);
    void
    add_write(const std::string& path, unsigned long long chunk);


    void add_value_iops(enum IopsOp);

    void
    add_value_size(enum SizeOp, unsigned long long value);

    double get_mean(enum IopsOp);


    double get_mean(enum SizeOp);

    std::vector<double> get_four_means(enum SizeOp);

    std::vector<double> get_four_means(enum IopsOp);
};

} // namespace gkfs::utils

#endif // GKFS_COMMON_STATS_HPP