Resolve "Client: Support MessagePack for process bandwidth monitoring"

Marc Vefrequested to merge
marc/271-client-support-messagepack-for-process-bandwidth-monitoring into master

GekkoFS clients support capturing the I/O traces of each individual process and periodically exporting them to a given file or ZeroMQ sink via the TCP protocol. To use this feature, the corresponding ZeroMQ (libzmq and cppzmq) dependencies are required which can be found in the default_zmq dependency profile. In addition, GekkoFS must be compiled with client metrics enabled (disabled by default) via the CMake argument -DGKFS_ENABLE_CLIENT_METRICS=ON.

Client metrics are individually enabled per GekkoFS client process via the following environment variables:

  • LIBGKFS_ENABLE_METRICS=ON enables capturing client-side metrics.
  • LIBGKFS_METRICS_FLUSH_INTERVAL=10 sets the flush interval to 10 seconds (defaults to 5). All outstanding client metrics are flushed when the process ends.
  • LIBGKFS_METRICS_PATH=<path> sets the path to flush client-metrics (defaults to /tmp/gkfs_client_metrics).
  • LIBGKFS_METRICS_IP_PORT=127.0.0.1:5555 enables flushing to a set ZeroMQ server. This option disables flushing to a file.

The ZeroMQ export can be tested via the gkfs_clientmetrics2json application which is built when enabling the CMake option -DGKFS_BUILD_TOOLS=ON:

  • Starting the ZeroMQ server: gkfs_clientmetrics2json tcp://127.0.0.1:5555
  • gkfs_clientmetrics2json <path> can also be used to unpack the Messagepack export from a file. Examplarily output with the ZeroMQ sink enabled when running: LD_PRELOAD=libgkfs_intercept.so LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_IP_PORT=127.0.0.1:5555 gkfs cp testfile /tmp/gkfs_mountdir/testfile:
~ $ gkfs_clientmetrics2json tcp://127.0.0.1:5555
Binding to: tcp://127.0.0.1:5555
Waiting for message...

Received message with size 68
Generated JSON:
[extra]avg_thruput_mib: [221.93,175.87,266.81,135.69]
end_t_micro: [8008,12396,16006,18454]
flush_t: 18564
hostname: "evie"
io_type: "w"
pid: 1259304
req_size: [524288,524288,524288,229502]
start_t_micro: [5755,9553,14132,16841]
total_bytes: 1802366
total_iops: 4

Closes #271 (closed)

Edited by Marc Vef

Merge request reports

CMake/gkfs-options.cmake

+16 −0
Changes for CMake/gkfs-options.cmake: 16 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -227,6 +227,13 @@ gkfs_define_option( 
  DEFAULT_VALUE OFF

)



# build tools

gkfs_define_option(

    GKFS_BUILD_TOOLS

    HELP_TEXT "Enable ${PROJECT_NAME} tools compilation"

    DEFAULT_VALUE OFF

)



cmake_dependent_option(GKFS_INSTALL_TESTS "Install GekkoFS self tests" OFF "GKFS_BUILD_TESTS" OFF)
@@ -356,4 +363,13 @@ gkfs_define_option( 
  DESCRIPTION "If GKFS_ENABLE_FORWARDING is ON, use AGIOS for scheduling I/Os"

)



################################################################################

# MSGPack client metrics

################################################################################



gkfs_define_option(

    GKFS_ENABLE_CLIENT_METRICS

    HELP_TEXT "Enable client metrics via MSGPack"

    DEFAULT_VALUE OFF

    DESCRIPTION "If GKFS_ENABLE_CLIENT_METRICS is ON, use MSGPack to dump client read/write metrics"

)
 
 No newline at end of file

CMake/load_nlohmann_json.cmake

0 → 100644
+52 −0
Changes for CMake/load_nlohmann_json.cmake: 52 added lines, 0 removed lines.
Original line number Diff line number Diff line 
################################################################################

# Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany          #

#                                                                              #

# This software was partially supported by the                                 #

# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #

#                                                                              #

# This software was partially supported by the                                 #

# ADA-FS project under the SPPEXA project funded by the DFG.                   #

#                                                                              #

# This file is part of GekkoFS.                                                #

#                                                                              #

# GekkoFS is free software: you can redistribute it and/or modify              #

# it under the terms of the GNU General Public License as published by         #

# the Free Software Foundation, either version 3 of the License, or            #

# (at your option) any later version.                                          #

#                                                                              #

# GekkoFS is distributed in the hope that it will be useful,                   #

# but WITHOUT ANY WARRANTY; without even the implied warranty of               #

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #

# GNU General Public License for more details.                                 #

#                                                                              #

# You should have received a copy of the GNU General Public License            #

# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #

#                                                                              #

# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################



include(FetchContent)



set(FETCHCONTENT_QUIET OFF)



FetchContent_Declare(nlohmann_json

    DOWNLOAD_EXTRACT_TIMESTAMP ON

    URL https://github.com/nlohmann/json/releases/download/v3.11.2/json.tar.xz)



FetchContent_GetProperties(nlohmann_json)



if (NOT nlohmann_json_POPULATED)

    FetchContent_Populate(nlohmann_json)

    message(STATUS "[gkfs.io] Nlohmann JSON source dir: ${nlohmann_json_SOURCE_DIR}")

    message(STATUS "[gkfs.io] Nlohmann JSON binary dir: ${nlohmann_json_BINARY_DIR}")



    # we don't really care so much about a third party library's tests to be

    # run from our own project's code

    set(JSON_BuildTests OFF CACHE INTERNAL "")



    # we also don't need to install it when our main project gets installed

    set(JSON_Install OFF CACHE INTERNAL "")



    add_subdirectory(${nlohmann_json_SOURCE_DIR} ${nlohmann_json_BINARY_DIR})

endif ()
 
 No newline at end of file

config/GekkoFWD/agios/agios.conf

+1 −1
Changes for config/GekkoFWD/agios/agios.conf: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ library_options: 
	#should the prediction module try to predict aggregations? (requires predict_read_traces = true)

	predict_request_aggregation = false ;



	#should the prediction module create simplified traces with information (the metrics) it obtained from the real traces?

	#should the prediction module create simplified traces with information (the client_metrics) it obtained from the real traces?

	predict_write_simplified_traces = false;



	#the tolerance for arrival times difference when checking if two predicted requests are the same (in %)

MessagePack @ 26527c71

Changes for external/MessagePack: 1 added line, 0 removed lines.
Original line number Diff line number Diff line 
Subproject commit 26527c71eb74fdc625ea994fa5caf45db318f7f1

include/client/env.hpp

+7 −0
Changes for include/client/env.hpp: 7 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -50,6 +50,13 @@ static constexpr auto LOG_OUTPUT_TRUNC = ADD_PREFIX("LOG_OUTPUT_TRUNC"); 
static constexpr auto CWD = ADD_PREFIX("CWD");

static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE");

static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE");

#ifdef GKFS_ENABLE_CLIENT_METRICS

static constexpr auto ENABLE_METRICS = ADD_PREFIX("ENABLE_METRICS");

static constexpr auto METRICS_FLUSH_INTERVAL =

        ADD_PREFIX("METRICS_FLUSH_INTERVAL");

static constexpr auto METRICS_PATH = ADD_PREFIX("METRICS_PATH");

static constexpr auto METRICS_IP_PORT = ADD_PREFIX("METRICS_IP_PORT");

#endif



static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL");

} // namespace gkfs::env