Commit 3a7c00de authored by Marc Vef's avatar Marc Vef
Browse files

Merge branch 'marc/271-client-support-messagepack-for-process-bandwidth-monitoring' into 'master' 

Resolve "Client: Support MessagePack for process bandwidth monitoring"

GekkoFS clients support capturing the I/O traces of each individual process and periodically exporting them to a given
file or ZeroMQ sink via the TCP protocol.
To use this feature, the corresponding ZeroMQ (`libzmq` and `cppzmq`) dependencies are required which can be found in
the `default_zmq` dependency profile.
In addition, GekkoFS must be compiled with client metrics enabled (disabled by default) via the CMake argument
`-DGKFS_ENABLE_CLIENT_METRICS=ON`.

Client metrics are individually enabled per GekkoFS client process via the following environment variables:

- `LIBGKFS_ENABLE_METRICS=ON` enables capturing client-side metrics.
- `LIBGKFS_METRICS_FLUSH_INTERVAL=10` sets the flush interval to 10 seconds (defaults to 5). All outstanding client
  metrics are flushed when the process ends.
- `LIBGKFS_METRICS_PATH=<path>` sets the path to flush client-metrics (defaults to `/tmp/gkfs_client_metrics`).
- `LIBGKFS_METRICS_IP_PORT=127.0.0.1:5555` enables flushing to a set ZeroMQ server. This option disables flushing to a
  file.

The ZeroMQ export can be tested via the `gkfs_clientmetrics2json` application which is built when enabling the CMake
option `-DGKFS_BUILD_TOOLS=ON`:

- Starting the ZeroMQ server: `gkfs_clientmetrics2json tcp://127.0.0.1:5555`
- `gkfs_clientmetrics2json <path>` can also be used to unpack the Messagepack export from a file.
  Examplarily output with the ZeroMQ sink enabled when running:
  `LD_PRELOAD=libgkfs_intercept.so LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_IP_PORT=127.0.0.1:5555 gkfs cp testfile /tmp/gkfs_mountdir/testfile`:

```bash
~ $ gkfs_clientmetrics2json tcp://127.0.0.1:5555
Binding to: tcp://127.0.0.1:5555
Waiting for message...

Received message with size 68
Generated JSON:
[extra]avg_thruput_mib: [221.93,175.87,266.81,135.69]
end_t_micro: [8008,12396,16006,18454]
flush_t: 18564
hostname: "evie"
io_type: "w"
pid: 1259304
req_size: [524288,524288,524288,229502]
start_t_micro: [5755,9553,14132,16841]
total_bytes: 1802366
total_iops: 4
```

Closes #271

Closes #271

See merge request !176
parents 1b23cfda b71f6229

.gitmodules

+3 −0
Original line number Diff line number Diff line
@@ -22,3 +22,6 @@ 
[submodule "external/CLI11"]

	path = external/CLI11

	url = https://github.com/CLIUtils/CLI11.git

[submodule "external/MessagePack"]

	path = external/MessagePack

	url = https://github.com/GekkoFS/MessagePackCPP.git

CHANGELOG.md

+17 −0
Original line number Diff line number Diff line
@@ -8,6 +8,20 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 
## [Unreleased]

### New



- Added client-side metrics including the periodic export to a file or ZeroMQ sink via the TCP

  protocol ([!176](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/176)).

  - CMake option added to enable this optional feature `-DGKFS_ENABLE_CLIENT_METRICS=ON`

  - A new CMake option `-DGKFS_BUILD_TOOLS=ON` was added which includes a ZeroMQ server to capture client-side metrics

  - The `libzmq` and `cppzmq` dependencies are required and can be found in the `default_zmq` profile.

  - Added new environment variables for the GekkoFS client:

    - `LIBGKFS_ENABLE_METRICS=ON` enables capturing client-side metrics

    - `LIBGKFS_METRICS_FLUSH_INTERVAL=10` sets the flush interval to 10 seconds (defaults to 5). All client metrics are

      flushed when the process ends.

    - `LIBGKFS_METRICS_PATH=<path>` sets the path to flush client-metrics (defaults to `/tmp/gkfs_client_metrics`).

    - `LIBGKFS_METRICS_IP_PORT=127.0.0.1:5555` enables flushing to a set ZeroMQ server. This option disables flushing to

      a file.

- Added the dependency profile for MOGON-NHR ([!176](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/176)).

- Added UCX and libfabric tcp support ([!176](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/176)).

- Added intercepton of `fadvise64()` and

  `fallocate()` ([!161](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/161)).

- Added user library `gkfs_user_lib` that can be used to directly link to an
@@ -31,6 +45,9 @@ replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141) 
### Removed

### Fixed



- An issue that updated the last modified time of a file during `stat` operations was

  fixed([!176](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/176)).



## [0.9.2] - 2024-02



### New

CMake/gkfs-options.cmake

+16 −0
Original line number Diff line number Diff line
@@ -227,6 +227,13 @@ gkfs_define_option( 
  DEFAULT_VALUE OFF

)



# build tools

gkfs_define_option(

    GKFS_BUILD_TOOLS

    HELP_TEXT "Enable ${PROJECT_NAME} tools compilation"

    DEFAULT_VALUE OFF

)



cmake_dependent_option(GKFS_INSTALL_TESTS "Install GekkoFS self tests" OFF "GKFS_BUILD_TESTS" OFF)
@@ -356,4 +363,13 @@ gkfs_define_option( 
  DESCRIPTION "If GKFS_ENABLE_FORWARDING is ON, use AGIOS for scheduling I/Os"

)



################################################################################

# MSGPack client metrics

################################################################################



gkfs_define_option(

    GKFS_ENABLE_CLIENT_METRICS

    HELP_TEXT "Enable client metrics via MSGPack"

    DEFAULT_VALUE OFF

    DESCRIPTION "If GKFS_ENABLE_CLIENT_METRICS is ON, use MSGPack to dump client read/write metrics"

)
 
 No newline at end of file

CMake/load_nlohmann_json.cmake

0 → 100644
+52 −0
Original line number Diff line number Diff line 
################################################################################

# Copyright 2018-2023, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2023, Johannes Gutenberg Universitaet Mainz, Germany          #

#                                                                              #

# This software was partially supported by the                                 #

# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #

#                                                                              #

# This software was partially supported by the                                 #

# ADA-FS project under the SPPEXA project funded by the DFG.                   #

#                                                                              #

# This file is part of GekkoFS.                                                #

#                                                                              #

# GekkoFS is free software: you can redistribute it and/or modify              #

# it under the terms of the GNU General Public License as published by         #

# the Free Software Foundation, either version 3 of the License, or            #

# (at your option) any later version.                                          #

#                                                                              #

# GekkoFS is distributed in the hope that it will be useful,                   #

# but WITHOUT ANY WARRANTY; without even the implied warranty of               #

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #

# GNU General Public License for more details.                                 #

#                                                                              #

# You should have received a copy of the GNU General Public License            #

# along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.            #

#                                                                              #

# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################



include(FetchContent)



set(FETCHCONTENT_QUIET OFF)



FetchContent_Declare(nlohmann_json

    DOWNLOAD_EXTRACT_TIMESTAMP ON

    URL https://github.com/nlohmann/json/releases/download/v3.11.2/json.tar.xz)



FetchContent_GetProperties(nlohmann_json)



if (NOT nlohmann_json_POPULATED)

    FetchContent_Populate(nlohmann_json)

    message(STATUS "[gkfs.io] Nlohmann JSON source dir: ${nlohmann_json_SOURCE_DIR}")

    message(STATUS "[gkfs.io] Nlohmann JSON binary dir: ${nlohmann_json_BINARY_DIR}")



    # we don't really care so much about a third party library's tests to be

    # run from our own project's code

    set(JSON_BuildTests OFF CACHE INTERNAL "")



    # we also don't need to install it when our main project gets installed

    set(JSON_Install OFF CACHE INTERNAL "")



    add_subdirectory(${nlohmann_json_SOURCE_DIR} ${nlohmann_json_BINARY_DIR})

endif ()
 
 No newline at end of file

CMakeLists.txt

+21 −1
Original line number Diff line number Diff line
@@ -31,7 +31,7 @@ cmake_minimum_required(VERSION 3.13) 


project(

    GekkoFS

    VERSION 0.9.2

    VERSION 0.9.3

    LANGUAGES ASM CXX C

)
@@ -167,6 +167,12 @@ if (GKFS_ENABLE_AGIOS) 
    find_package(AGIOS REQUIRED)

endif()



if (GKFS_ENABLE_CLIENT_METRICS)

    ### zeromq: required for sending client metrics to FTIO

    message(STATUS "[${PROJECT_NAME}] Checking for cppzmq")

    find_package(cppzmq REQUIRED)

endif ()



### Metadata backends

if(GKFS_ENABLE_ROCKSDB)

    message(STATUS "[${PROJECT_NAME}] Checking for RocksDB")
@@ -218,6 +224,7 @@ find_package(Filesystem REQUIRED) 
set(GKFS_DEPENDENCIES_PATH ${CMAKE_SOURCE_DIR}/external)



### {fmt}: required for sensible output formatting

set(FMT_INSTALL OFF)

include_from_source(fmt

  MESSAGE "[${PROJECT_NAME}] Searching for {fmt}"

  SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/fmt
@@ -244,6 +251,15 @@ include_from_source(cli11 
  GIT_TAG v2.2.0

  )



if (GKFS_ENABLE_CLIENT_METRICS)

    ### MessagePack: used for monitoring information on the client

    include_from_source(MessagePack

        MESSAGE "[${PROJECT_NAME}] Searching for MessagePackCPP"

        SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/MessagePack

        GIT_REPOSITORY https://github.com/GekkoFS/MessagePackCPP

        GIT_TAG 51655ec8f2b8fed84f685e3bcf8f226b139a263b # latest HEAD

    )

endif ()



################################################################################

## Check configured variables/options and act accordingly
@@ -302,11 +318,15 @@ set(INCLUDE_DIR "${CMAKE_SOURCE_DIR}/include") 
include_directories(

    ${INCLUDE_DIR}

    ${CMAKE_BINARY_DIR}/include

    /usr/include

)



add_subdirectory(src)

add_subdirectory(include)

add_subdirectory(examples)

if (GKFS_BUILD_TOOLS)

    add_subdirectory(tools)

endif ()



### Mark any CMake variables imported from {fmt} and spdlog as advanced, so

### that they don't appear in cmake-gui or ccmake. Similarly for FETCHCONTENT