Loading .gitlab-ci.yml +2 −2 Original line number Diff line number Diff line Loading @@ -25,8 +25,8 @@ variables: GIT_SUBMODULE_STRATEGY: recursive CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" IMAGE_PREFIX: "${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/" #IMAGE_PREFIX: "gekkofs/" #IMAGE_PREFIX: "${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/" IMAGE_PREFIX: "gekkofs/" CORE: "${IMAGE_PREFIX}core:${VERSION}" LINTER: "${IMAGE_PREFIX}linter:${VERSION}" Loading README.md +15 −0 Original line number Diff line number Diff line Loading @@ -961,11 +961,26 @@ Environment variable prefixes are scoped by component: - `GKFS_DAEMON_ENABLE_CHUNKSTATS` - Enable chunk statistics collection (default: OFF). - `GKFS_DAEMON_ENABLE_PROMETHEUS` - Enable Prometheus output when compiled with `GKFS_ENABLE_PROMETHEUS` (default: OFF). - `GKFS_DAEMON_PROMETHEUS_GATEWAY` - Prometheus Pushgateway address, `host:port`. - `GKFS_DAEMON_STATS_FILE` - Write periodic statistics to this file. - `GKFS_DAEMON_STATS_LOG` - Write periodic statistics to the daemon log stream (default: OFF). - `GKFS_DAEMON_MIGRATION_CHECKPOINT` - Enable atomic per-daemon migration checkpoints (default: OFF). These variables are read once during daemon startup. They add no statistics collection overhead when unset. Existing command-line options remain supported. Periodic snapshots include a wall-clock `timestamp_ms`, daemon `pid`, `uptime_seconds`, sampled Argobots I/O queue depth (`io_queue_depth`) and total queued tasks (`io_queue_total`), plus migration job and byte counters. The same runtime values are exposed to Prometheus in the `GKFS_RUNTIME` gauge family with a low-cardinality `metric` label. File and daemon-log snapshots contain one `METRICS_JSON { ... }` record per sample, followed by the readable statistics report. The JSON record includes RPC request/error totals and latency maxima, backend read/write/error/retry totals and latency maxima, migration progress, queue depth, and cache fields. Client MessagePack metrics additionally report `cache_hits` and `cache_misses` for client-side dentry-cache activity. ### Client #### Core - `LIBGKFS_HOSTS_FILE` - Path to the hostsfile (created by the daemon and mandatory for the client). Loading WIP.md +6 −2 Original line number Diff line number Diff line Loading @@ -310,8 +310,12 @@ readiness and richer lifecycle/health reporting remain future work. ### 15. Improve metrics and tracing coverage **Status: Partial.** Existing daemon metrics and migration checkpoints are explicitly startup-opt-in through `GKFS_DAEMON_*` environment variables; broader stable metrics and distributed tracing remain. explicitly startup-opt-in through `GKFS_DAEMON_*` environment variables. Timestamped snapshots now include sampled I/O queue depth and migration counters in file/log output and Prometheus gauges. RPC and backend error, retry, and latency aggregates are included, and client MessagePack metrics include dentry-cache hits/misses. Broader per-operation metrics and distributed tracing remain. Define stable, low-cardinality metrics for operation count/bytes/latency/errors; RPC queues, retries, and timeouts; backend latency and cache hits; migration and Loading docs/sphinx/users/running.md +11 −1 Original line number Diff line number Diff line Loading @@ -293,10 +293,20 @@ environment variables: `GKFS_DAEMON_ENABLE_STATS=ON`, `GKFS_DAEMON_ENABLE_CHUNKSTATS=ON`, and `GKFS_DAEMON_ENABLE_PROMETHEUS=ON`. Set `GKFS_DAEMON_PROMETHEUS_GATEWAY=host:port` to override the Pushgateway. When these variables are unset, no statistics collection thread is started. Set `GKFS_DAEMON_STATS_FILE=/path/to/stats.log` for periodic file output, or `GKFS_DAEMON_STATS_LOG=ON` for periodic output to the daemon log stream. When all these variables are unset, no statistics collection thread is started. Set `GKFS_DAEMON_MIGRATION_CHECKPOINT=ON` to enable atomic per-daemon migration checkpoint files; this is startup-only and disabled by default. Snapshots include `timestamp_ms`, `pid`, `uptime_seconds`, sampled I/O queue depth/total, and migration job/byte counters. Prometheus exposes the runtime values as `GKFS_RUNTIME{metric="..."}` gauges. File and daemon-log snapshots also contain a `METRICS_JSON { ... }` record with RPC latency/errors, backend latency/errors/retries, and migration fields before the human-readable report. Client MessagePack output includes cache hit/miss counters. ### Advanced experimental features #### Rename Loading include/common/statistics/stats.hpp +69 −3 Original line number Diff line number Diff line Loading @@ -53,14 +53,17 @@ #include <iostream> #include <iomanip> #include <fstream> #include <sstream> #include <atomic> #include <mutex> #include <functional> #include <config.hpp> // PROMETHEUS includes #ifdef GKFS_ENABLE_PROMETHEUS #include <prometheus/counter.h> #include <prometheus/gauge.h> #include <prometheus/summary.h> #include <prometheus/exposer.h> #include <prometheus/registry.h> Loading @@ -78,6 +81,30 @@ using namespace prometheus; */ namespace gkfs::utils { struct RuntimeMetrics { uint64_t io_queue_depth{0}; uint64_t io_queue_total{0}; uint64_t migration_jobs_total{0}; uint64_t migration_jobs_completed{0}; uint64_t migration_jobs_succeeded{0}; uint64_t migration_jobs_failed{0}; uint64_t migration_bytes_transferred{0}; uint64_t rpc_requests{0}; uint64_t rpc_errors{0}; uint64_t rpc_latency_us_total{0}; uint64_t rpc_latency_us_max{0}; uint64_t backend_read_latency_us_total{0}; uint64_t backend_write_latency_us_total{0}; uint64_t backend_read_latency_us_max{0}; uint64_t backend_write_latency_us_max{0}; uint64_t backend_reads{0}; uint64_t backend_writes{0}; uint64_t backend_errors{0}; uint64_t backend_retries{0}; uint64_t cache_hits{0}; uint64_t cache_misses{0}; }; /** * * Number of operations (Create, write/ read, remove, mkdir...) Loading Loading @@ -153,6 +180,22 @@ private: bool output_thread_; ///< Enables or disables the output thread bool enable_prometheus_; ///< Enables or disables the prometheus output bool enable_chunkstats_; ///< Enables or disables the chunk stats output bool enable_log_; ///< Enables periodic output to the daemon log std::function<RuntimeMetrics()> runtime_metrics_; std::atomic<uint64_t> rpc_requests_{0}; std::atomic<uint64_t> rpc_errors_{0}; std::atomic<uint64_t> rpc_latency_us_total_{0}; std::atomic<uint64_t> rpc_latency_us_max_{0}; std::atomic<uint64_t> backend_read_latency_us_total_{0}; std::atomic<uint64_t> backend_write_latency_us_total_{0}; std::atomic<uint64_t> backend_read_latency_us_max_{0}; std::atomic<uint64_t> backend_write_latency_us_max_{0}; std::atomic<uint64_t> backend_reads_{0}; std::atomic<uint64_t> backend_writes_{0}; std::atomic<uint64_t> backend_errors_{0}; std::atomic<uint64_t> backend_retries_{0}; std::atomic<uint64_t> cache_hits_{0}; std::atomic<uint64_t> cache_misses_{0}; bool running = Loading Loading @@ -188,9 +231,25 @@ private: * @param of Output stream */ void dump(std::ofstream& of); dump(std::ostream& of, const RuntimeMetrics& runtime_metrics, uint64_t timestamp_ms); public: void record_rpc(uint64_t latency_us, bool error); void record_backend_read(bool error, uint64_t retries = 0, uint64_t latency_us = 0); void record_backend_write(bool error, uint64_t retries = 0, uint64_t latency_us = 0); void record_cache(bool hit); private: // Prometheus Push structs #ifdef GKFS_ENABLE_PROMETHEUS std::shared_ptr<Gateway> gateway; ///< Prometheus Gateway Loading @@ -199,8 +258,14 @@ private: ///< Prometheus cpp) Family<Summary>* family_summary; ///< Prometheus SIZE counter (managed by ///< Prometheus cpp) Family<Counter>* family_runtime_counter; Family<Gauge>* family_runtime_gauge; Family<Summary>* family_runtime_summary; std::map<IopsOp, Counter*> iops_prometheus; ///< Prometheus IOPS metrics std::map<SizeOp, Summary*> size_prometheus; ///< Prometheus SIZE metrics std::map<std::string, Counter*> runtime_counters_prometheus; std::map<std::string, Gauge*> runtime_gauges_prometheus; std::map<std::string, Summary*> runtime_summaries_prometheus; #endif public: Loading @@ -211,8 +276,9 @@ public: * @param filename file where to write the output * @param prometheus_gateway ip:port to expose the metrics */ Stats(bool enable_chunkstats, bool enable_prometheus, const std::string& filename, const std::string& prometheus_gateway); Stats(bool enable_chunkstats, bool enable_prometheus, bool enable_log, const std::string& filename, const std::string& prometheus_gateway, std::function<RuntimeMetrics()> runtime_metrics = nullptr); /** * @brief Destroys the class, and any associated thread Loading Loading
.gitlab-ci.yml +2 −2 Original line number Diff line number Diff line Loading @@ -25,8 +25,8 @@ variables: GIT_SUBMODULE_STRATEGY: recursive CCACHE_DIR: "${CI_PROJECT_DIR}/ccache" IMAGE_PREFIX: "${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/" #IMAGE_PREFIX: "gekkofs/" #IMAGE_PREFIX: "${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/" IMAGE_PREFIX: "gekkofs/" CORE: "${IMAGE_PREFIX}core:${VERSION}" LINTER: "${IMAGE_PREFIX}linter:${VERSION}" Loading
README.md +15 −0 Original line number Diff line number Diff line Loading @@ -961,11 +961,26 @@ Environment variable prefixes are scoped by component: - `GKFS_DAEMON_ENABLE_CHUNKSTATS` - Enable chunk statistics collection (default: OFF). - `GKFS_DAEMON_ENABLE_PROMETHEUS` - Enable Prometheus output when compiled with `GKFS_ENABLE_PROMETHEUS` (default: OFF). - `GKFS_DAEMON_PROMETHEUS_GATEWAY` - Prometheus Pushgateway address, `host:port`. - `GKFS_DAEMON_STATS_FILE` - Write periodic statistics to this file. - `GKFS_DAEMON_STATS_LOG` - Write periodic statistics to the daemon log stream (default: OFF). - `GKFS_DAEMON_MIGRATION_CHECKPOINT` - Enable atomic per-daemon migration checkpoints (default: OFF). These variables are read once during daemon startup. They add no statistics collection overhead when unset. Existing command-line options remain supported. Periodic snapshots include a wall-clock `timestamp_ms`, daemon `pid`, `uptime_seconds`, sampled Argobots I/O queue depth (`io_queue_depth`) and total queued tasks (`io_queue_total`), plus migration job and byte counters. The same runtime values are exposed to Prometheus in the `GKFS_RUNTIME` gauge family with a low-cardinality `metric` label. File and daemon-log snapshots contain one `METRICS_JSON { ... }` record per sample, followed by the readable statistics report. The JSON record includes RPC request/error totals and latency maxima, backend read/write/error/retry totals and latency maxima, migration progress, queue depth, and cache fields. Client MessagePack metrics additionally report `cache_hits` and `cache_misses` for client-side dentry-cache activity. ### Client #### Core - `LIBGKFS_HOSTS_FILE` - Path to the hostsfile (created by the daemon and mandatory for the client). Loading
WIP.md +6 −2 Original line number Diff line number Diff line Loading @@ -310,8 +310,12 @@ readiness and richer lifecycle/health reporting remain future work. ### 15. Improve metrics and tracing coverage **Status: Partial.** Existing daemon metrics and migration checkpoints are explicitly startup-opt-in through `GKFS_DAEMON_*` environment variables; broader stable metrics and distributed tracing remain. explicitly startup-opt-in through `GKFS_DAEMON_*` environment variables. Timestamped snapshots now include sampled I/O queue depth and migration counters in file/log output and Prometheus gauges. RPC and backend error, retry, and latency aggregates are included, and client MessagePack metrics include dentry-cache hits/misses. Broader per-operation metrics and distributed tracing remain. Define stable, low-cardinality metrics for operation count/bytes/latency/errors; RPC queues, retries, and timeouts; backend latency and cache hits; migration and Loading
docs/sphinx/users/running.md +11 −1 Original line number Diff line number Diff line Loading @@ -293,10 +293,20 @@ environment variables: `GKFS_DAEMON_ENABLE_STATS=ON`, `GKFS_DAEMON_ENABLE_CHUNKSTATS=ON`, and `GKFS_DAEMON_ENABLE_PROMETHEUS=ON`. Set `GKFS_DAEMON_PROMETHEUS_GATEWAY=host:port` to override the Pushgateway. When these variables are unset, no statistics collection thread is started. Set `GKFS_DAEMON_STATS_FILE=/path/to/stats.log` for periodic file output, or `GKFS_DAEMON_STATS_LOG=ON` for periodic output to the daemon log stream. When all these variables are unset, no statistics collection thread is started. Set `GKFS_DAEMON_MIGRATION_CHECKPOINT=ON` to enable atomic per-daemon migration checkpoint files; this is startup-only and disabled by default. Snapshots include `timestamp_ms`, `pid`, `uptime_seconds`, sampled I/O queue depth/total, and migration job/byte counters. Prometheus exposes the runtime values as `GKFS_RUNTIME{metric="..."}` gauges. File and daemon-log snapshots also contain a `METRICS_JSON { ... }` record with RPC latency/errors, backend latency/errors/retries, and migration fields before the human-readable report. Client MessagePack output includes cache hit/miss counters. ### Advanced experimental features #### Rename Loading
include/common/statistics/stats.hpp +69 −3 Original line number Diff line number Diff line Loading @@ -53,14 +53,17 @@ #include <iostream> #include <iomanip> #include <fstream> #include <sstream> #include <atomic> #include <mutex> #include <functional> #include <config.hpp> // PROMETHEUS includes #ifdef GKFS_ENABLE_PROMETHEUS #include <prometheus/counter.h> #include <prometheus/gauge.h> #include <prometheus/summary.h> #include <prometheus/exposer.h> #include <prometheus/registry.h> Loading @@ -78,6 +81,30 @@ using namespace prometheus; */ namespace gkfs::utils { struct RuntimeMetrics { uint64_t io_queue_depth{0}; uint64_t io_queue_total{0}; uint64_t migration_jobs_total{0}; uint64_t migration_jobs_completed{0}; uint64_t migration_jobs_succeeded{0}; uint64_t migration_jobs_failed{0}; uint64_t migration_bytes_transferred{0}; uint64_t rpc_requests{0}; uint64_t rpc_errors{0}; uint64_t rpc_latency_us_total{0}; uint64_t rpc_latency_us_max{0}; uint64_t backend_read_latency_us_total{0}; uint64_t backend_write_latency_us_total{0}; uint64_t backend_read_latency_us_max{0}; uint64_t backend_write_latency_us_max{0}; uint64_t backend_reads{0}; uint64_t backend_writes{0}; uint64_t backend_errors{0}; uint64_t backend_retries{0}; uint64_t cache_hits{0}; uint64_t cache_misses{0}; }; /** * * Number of operations (Create, write/ read, remove, mkdir...) Loading Loading @@ -153,6 +180,22 @@ private: bool output_thread_; ///< Enables or disables the output thread bool enable_prometheus_; ///< Enables or disables the prometheus output bool enable_chunkstats_; ///< Enables or disables the chunk stats output bool enable_log_; ///< Enables periodic output to the daemon log std::function<RuntimeMetrics()> runtime_metrics_; std::atomic<uint64_t> rpc_requests_{0}; std::atomic<uint64_t> rpc_errors_{0}; std::atomic<uint64_t> rpc_latency_us_total_{0}; std::atomic<uint64_t> rpc_latency_us_max_{0}; std::atomic<uint64_t> backend_read_latency_us_total_{0}; std::atomic<uint64_t> backend_write_latency_us_total_{0}; std::atomic<uint64_t> backend_read_latency_us_max_{0}; std::atomic<uint64_t> backend_write_latency_us_max_{0}; std::atomic<uint64_t> backend_reads_{0}; std::atomic<uint64_t> backend_writes_{0}; std::atomic<uint64_t> backend_errors_{0}; std::atomic<uint64_t> backend_retries_{0}; std::atomic<uint64_t> cache_hits_{0}; std::atomic<uint64_t> cache_misses_{0}; bool running = Loading Loading @@ -188,9 +231,25 @@ private: * @param of Output stream */ void dump(std::ofstream& of); dump(std::ostream& of, const RuntimeMetrics& runtime_metrics, uint64_t timestamp_ms); public: void record_rpc(uint64_t latency_us, bool error); void record_backend_read(bool error, uint64_t retries = 0, uint64_t latency_us = 0); void record_backend_write(bool error, uint64_t retries = 0, uint64_t latency_us = 0); void record_cache(bool hit); private: // Prometheus Push structs #ifdef GKFS_ENABLE_PROMETHEUS std::shared_ptr<Gateway> gateway; ///< Prometheus Gateway Loading @@ -199,8 +258,14 @@ private: ///< Prometheus cpp) Family<Summary>* family_summary; ///< Prometheus SIZE counter (managed by ///< Prometheus cpp) Family<Counter>* family_runtime_counter; Family<Gauge>* family_runtime_gauge; Family<Summary>* family_runtime_summary; std::map<IopsOp, Counter*> iops_prometheus; ///< Prometheus IOPS metrics std::map<SizeOp, Summary*> size_prometheus; ///< Prometheus SIZE metrics std::map<std::string, Counter*> runtime_counters_prometheus; std::map<std::string, Gauge*> runtime_gauges_prometheus; std::map<std::string, Summary*> runtime_summaries_prometheus; #endif public: Loading @@ -211,8 +276,9 @@ public: * @param filename file where to write the output * @param prometheus_gateway ip:port to expose the metrics */ Stats(bool enable_chunkstats, bool enable_prometheus, const std::string& filename, const std::string& prometheus_gateway); Stats(bool enable_chunkstats, bool enable_prometheus, bool enable_log, const std::string& filename, const std::string& prometheus_gateway, std::function<RuntimeMetrics()> runtime_metrics = nullptr); /** * @brief Destroys the class, and any associated thread Loading