Commit 82db825f authored by Ramon Nou's avatar Ramon Nou
Browse files

Stabilize stats snapshot test under TSAN

parent 6767fac8

include/common/statistics/stats.hpp

+4 −2
Original line number Diff line number Diff line
@@ -202,6 +202,7 @@ private: 
    bool enable_chunkstats_; ///< Enables or disables the chunk stats output

    bool enable_log_;        ///< Enables periodic output to the daemon log

    std::function<RuntimeMetrics()> runtime_metrics_;

    std::chrono::milliseconds output_interval_{std::chrono::seconds(10)};

    std::atomic<uint64_t> rpc_requests_{0};

    std::atomic<uint64_t> rpc_errors_{0};

    std::atomic<uint64_t> rpc_latency_us_total_{0};
@@ -226,7 +227,7 @@ private: 
     * @param file_output is the output file

     */

    void

    output(std::chrono::seconds d, std::string file_output);

    output(std::chrono::milliseconds d, std::string file_output);



    std::map<std::pair<std::string, unsigned long long>,

             std::atomic<unsigned int>>
@@ -306,7 +307,8 @@ public: 
     */

    Stats(bool enable_chunkstats, bool enable_prometheus, bool enable_log,

          const std::string& filename, const std::string& prometheus_gateway,

          std::function<RuntimeMetrics()> runtime_metrics = nullptr);

          std::function<RuntimeMetrics()> runtime_metrics = nullptr,

          std::chrono::milliseconds output_interval = std::chrono::seconds(10));



    /**

     * @brief Destroys the class, and any associated thread

src/common/statistics/stats.cpp

+14 −11
Original line number Diff line number Diff line
@@ -140,10 +140,12 @@ Stats::setup_Prometheus(const std::string& gateway_ip, 
Stats::Stats(bool enable_chunkstats, bool enable_prometheus, bool enable_log,

             const std::string& stats_file,

             const std::string& prometheus_gateway,

             std::function<RuntimeMetrics()> runtime_metrics)

             std::function<RuntimeMetrics()> runtime_metrics,

             const std::chrono::milliseconds output_interval)

    : enable_prometheus_(enable_prometheus),

      enable_chunkstats_(enable_chunkstats), enable_log_(enable_log),

      runtime_metrics_(std::move(runtime_metrics)) {

      runtime_metrics_(std::move(runtime_metrics)),

      output_interval_(output_interval) {



    // Init clocks

    start = std::chrono::steady_clock::now();
@@ -172,9 +174,8 @@ Stats::Stats(bool enable_chunkstats, bool enable_prometheus, bool enable_log, 
    output_thread_ = false;

    if(!stats_file.empty() || enable_prometheus_ || enable_log_) {

        output_thread_ = true;

        t_output = std::thread([this, stats_file] {

            output(std::chrono::duration(10s), stats_file);

        });

        t_output = std::thread(

                [this, stats_file] { output(output_interval_, stats_file); });

    }

}
@@ -563,7 +564,7 @@ Stats::dump(std::ostream& of, const RuntimeMetrics& runtime_metrics, 
    of << std::endl;

}

void

Stats::output(std::chrono::seconds d, std::string file_output) {

Stats::output(std::chrono::milliseconds d, std::string file_output) {

    int times = 0;

    std::optional<std::ofstream> of;

    if(!file_output.empty())
@@ -673,6 +674,7 @@ Stats::output(std::chrono::seconds d, std::string file_output) { 
                << runtime_metrics.materialization_failures

                << ",\"materialized_bytes\":"

                << runtime_metrics.materialized_bytes << "}\n";

            of->flush();

        }

        if(enable_log_) {

            std::ostringstream metrics;
@@ -791,8 +793,6 @@ Stats::output(std::chrono::seconds d, std::string file_output) { 
                    ->Set(runtime_metrics.backend_write_latency_us_max);

        }

#endif

        std::chrono::seconds a = 0s;



        times++;



        if(enable_chunkstats_ && of) {
@@ -804,9 +804,12 @@ Stats::output(std::chrono::seconds d, std::string file_output) { 
            gateway->Push();

        }

#endif

        while(running && a < d) {

            a += 1s;

            std::this_thread::sleep_for(1s);

        auto remaining = d;

        while(running && remaining > 0ms) {

            const auto sleep_duration =

                    std::min(remaining, std::chrono::milliseconds(1000));

            std::this_thread::sleep_for(sleep_duration);

            remaining -= sleep_duration;

        }

    }

}

tests/unit/test_stats.cpp

+4 −2
Original line number Diff line number Diff line
@@ -57,7 +57,7 @@ TEST_CASE("Stats file output includes a complete METRICS_JSON runtime snapshot", 
    {

        gkfs::utils::Stats stats(

                false, false, false, output_file.string(), "localhost:9091",

                [supplied] { return supplied; });

                [supplied] { return supplied; }, std::chrono::milliseconds(10));

        stats.record_rpc(17, true);

        stats.record_backend_read(false, 2, 23);

        stats.record_backend_write(true, 1, 29);
@@ -74,7 +74,9 @@ TEST_CASE("Stats file output includes a complete METRICS_JSON runtime snapshot", 
                              std::chrono::seconds(2);

        while(std::chrono::steady_clock::now() < deadline) {

            const auto contents = read_file(output_file);

            if(contents.find("METRICS_JSON {") != std::string::npos) {

            if(contents.find("\"operations_stat\":1") !=

                       std::string::npos &&

               contents.find("\"bytes_read\":64") != std::string::npos) {

                break;

            }

            std::this_thread::sleep_for(std::chrono::milliseconds(10));