Added GekkoFS integration with Lustre's hierarchical storage management

FindCargo

---------

Find Cargo include dirs and libraries.

Use this module by invoking find_package with the form::

  find_package(Cargo

    [version] [EXACT]     # Minimum or EXACT version e.g. 0.6.2

    [REQUIRED]            # Fail with error if Cargo is not found

    )

Imported Targets

^^^^^^^^^^^^^^^^

This module provides the following imported targets, if found:

``Cargo::Cargo``

  The Cargo library

Result Variables

^^^^^^^^^^^^^^^^

This will define the following variables:

``Cargo_FOUND``

  True if the system has the Cargo library.

``Cargo_VERSION``

  The version of the Cargo library which was found.

``Cargo_INCLUDE_DIRS``

  Include directories needed to use Cargo.

``Cargo_LIBRARIES``

  Libraries needed to link to Cargo.

Cache Variables

^^^^^^^^^^^^^^^

The following cache variables may also be set:

``CARGO_INCLUDE_DIR``

  The directory containing ``cargo.hpp``.

``CARGO_LIBRARY``

  The path to the Cargo library.

#]=======================================================================]

function(_get_pkgconfig_paths target_var)

  set(_lib_dirs)

  if(NOT DEFINED CMAKE_SYSTEM_NAME

     OR (CMAKE_SYSTEM_NAME MATCHES "^(Linux|kFreeBSD|GNU)$"

         AND NOT CMAKE_CROSSCOMPILING)

  )

    if(EXISTS "/etc/debian_version") # is this a debian system ?

      if(CMAKE_LIBRARY_ARCHITECTURE)

        list(APPEND _lib_dirs "lib/${CMAKE_LIBRARY_ARCHITECTURE}/pkgconfig")

      endif()

    else()

      # not debian, check the FIND_LIBRARY_USE_LIB32_PATHS and FIND_LIBRARY_USE_LIB64_PATHS properties

      get_property(uselib32 GLOBAL PROPERTY FIND_LIBRARY_USE_LIB32_PATHS)

      if(uselib32 AND CMAKE_SIZEOF_VOID_P EQUAL 4)

        list(APPEND _lib_dirs "lib32/pkgconfig")

      endif()

      get_property(uselib64 GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS)

      if(uselib64 AND CMAKE_SIZEOF_VOID_P EQUAL 8)

        list(APPEND _lib_dirs "lib64/pkgconfig")

      endif()

      get_property(uselibx32 GLOBAL PROPERTY FIND_LIBRARY_USE_LIBX32_PATHS)

      if(uselibx32 AND CMAKE_INTERNAL_PLATFORM_ABI STREQUAL "ELF X32")

        list(APPEND _lib_dirs "libx32/pkgconfig")

      endif()

    endif()

  endif()

  if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD" AND NOT CMAKE_CROSSCOMPILING)

    list(APPEND _lib_dirs "libdata/pkgconfig")

  endif()

  list(APPEND _lib_dirs "lib/pkgconfig")

  list(APPEND _lib_dirs "share/pkgconfig")

  set(_extra_paths)

  list(APPEND _extra_paths ${CMAKE_PREFIX_PATH})

  list(APPEND _extra_paths ${CMAKE_FRAMEWORK_PATH})

  list(APPEND _extra_paths ${CMAKE_APPBUNDLE_PATH})

  # Check if directories exist and eventually append them to the

  # pkgconfig path list

  foreach(_prefix_dir ${_extra_paths})

    foreach(_lib_dir ${_lib_dirs})

      if(EXISTS "${_prefix_dir}/${_lib_dir}")

        list(APPEND _pkgconfig_paths "${_prefix_dir}/${_lib_dir}")

        list(REMOVE_DUPLICATES _pkgconfig_paths)

      endif()

    endforeach()

  endforeach()

  set("${target_var}"

      ${_pkgconfig_paths}

      PARENT_SCOPE

  )

endfunction()

# prevent repeating work if the main CMakeLists.txt already called

# find_package(PkgConfig)

if(NOT PKG_CONFIG_FOUND)

  find_package(PkgConfig)

endif()

if(PKG_CONFIG_FOUND)

  pkg_check_modules(PC_CARGO QUIET cargo)

  find_path(

    CARGO_INCLUDE_DIR

    NAMES cargo.hpp

    PATHS ${PC_CARGO_INCLUDE_DIRS}

    PATH_SUFFIXES include/cargo

  )

  find_library(

    CARGO_LIBRARY

    NAMES cargo

    PATHS ${PC_CARGO_LIBRARY_DIRS}

  )

  set(Cargo_VERSION ${PC_CARGO_VERSION})

else()

  find_path(

    CARGO_INCLUDE_DIR

    NAMES cargo.hpp

    PATH_SUFFIXES include/cargo

  )

  find_library(CARGO_LIBRARY NAMES cargo)

  # even if pkg-config is not available, but Cargo still installs a .pc file

  # that we can use to retrieve library information from. Try to find it at all

  # possible pkgconfig subfolders (depending on the system).

  _get_pkgconfig_paths(_pkgconfig_paths)

  find_file(_cargo_pc_file cargo.pc PATHS "${_pkgconfig_paths}")

  if(NOT _cargo_pc_file)

    message(

      FATAL_ERROR

        "ERROR: Could not find 'cargo.pc' file. Unable to determine library version"

    )

  endif()

  file(STRINGS "${_cargo_pc_file}" _cargo_pc_file_contents REGEX "Version: ")

  if("${_cargo_pc_file_contents}" MATCHES "Version: ([0-9]+\\.[0-9]+\\.[0-9])")

    set(Cargo_VERSION ${CMAKE_MATCH_1})

  else()

    message(FATAL_ERROR "ERROR: Failed to determine library version")

  endif()

  unset(_pkg_config_paths)

  unset(_cargo_pc_file_contents)

endif()

mark_as_advanced(Cargo_INCLUDE_DIR CARGO_LIBRARY)

include(FindPackageHandleStandardArgs)

find_package_handle_standard_args(

  Cargo

  FOUND_VAR Cargo_FOUND

  REQUIRED_VARS CARGO_LIBRARY CARGO_INCLUDE_DIR

  VERSION_VAR Cargo_VERSION

)

if(Cargo_FOUND)

  set(Cargo_INCLUDE_DIRS ${CARGO_INCLUDE_DIR})

  set(Cargo_LIBRARIES ${CARGO_LIBRARY})

  if(NOT TARGET Cargo::Cargo)

    add_library(Cargo::Cargo UNKNOWN IMPORTED)

    set_target_properties(

      Cargo::Cargo

      PROPERTIES IMPORTED_LOCATION "${CARGO_LIBRARY}"

                 INTERFACE_INCLUDE_DIRECTORIES "${CARGO_INCLUDE_DIR}"

    )

  endif()

endif()

  - [Server-side statistics via Prometheus](#server-side-statistics-via-prometheus)

  - [GekkoFS proxy](#gekkofs-proxy)

  - [File system expansion](#file-system-expansion)

  - [Lustre hierarchical storage management (HSM) integration](#luster-hierarchical-storage-management-hsm-integration)

- [Miscellaneous](#miscellaneous)

  - [External functions](#external-functions)

  - [Data placement](#data-placement)

* [gkfs] Shutdown time: 1.032 seconds

```

## Lustre hierarchical storage management (HSM) integration

HSM is part of Lustre, which can manage the transferred data on Lustre. With an HSM integration, it can transparently

transfer data between Lustre and GekkoFS. When a job starts, it receives a config file from the user and starts to

transfer files from Lustre to GekkoFS. When the job is done or another process from Lustre tries to access a transferred

file from Lustre, it flushes back the data to Lustre to maintain consistency. It can also let the user run a separate

job with shared data on two different filesystems, like Lustre as a backend filesystem and GekkoFS as a burst buffer

cache.

Checkout branch `lustre-hsm-integration` first.

For running, it is necessary to run GekkoFS daemons with administrative permissions. It sends a request to HSM API and

without that it cannot process the request. It is also necessary to start Cargo (data stager) prior to run GekkoFS.

#### Dependencies

- Transfer tool: Cargo

- Lustre API library. Using the API requires super user permissions

- Lustre version: 2.13.0+

#### Lustre API

Working with these functions are necessary for working copy manager with Lustre HSM (with sudo permissions):

```c++

llapi_hsm_copytool_unregister()

llapi_hsm_unregister_event_fifo()

llapi_hsm_register_event_fifo()

llapi_hsm_copytool_register()

```

These functions from Lustre API are also used for managing files on Lustre side (also with sudo permissions):

```c++

llapi_search_fsname()

//Recieves notification from Lustre HSM

llapi_hsm_copytool_recv()

//Notifies HSM Lustre about the progress of file transfer

llapi_hsm_action_progress()

//Notifies HSM Lustre about the beginning of file transfer 

llapi_hsm_action_begin()

//Notifies HSM Lustre about the completion of file transfer

llapi_hsm_action_end()

//Recieves file descriptor from HSM 

llapi_hsm_action_get_fd()

//Convert fid to path

llapi_fid2path()

//send HSM request

llapi_hsm_request() 

```

#### Installing Cargo

1. Use the scripts on the script directory to install the Cargo dependencies first

2. Install Cargo:

```bash

cmake \

-DCMAKE_PREFIX_PATH:STRING="$CMAKE_PREFIX_PATH" \

-DCMAKE_INSTALL_PREFIX:STRING="$INSTALL_PATH" \

-DCARGO_TRANSPORT_LIBRARY:STRING=libfabric \

-DCARGO_BUILD_TESTS:BOOL=ON \

..

```

#### Running Cargo

Run the Cargo on the selected address:

```bash

${INSTALL_DIR}/bin/cargo -l ofi+tcp://127.0.0.1:52000

```

This command runs cargo on the nodes but GekkoFS needs to have this address to dispatch transfer requests to cargo

Update the /etc/config.conf file with the same address

```txt

ofi+tcp://127.0.0.1:52000

```

#### The user cache preference for HSM hsm_extension

The user can edit the existing file `path.conf` in the `etc` directory to add the preferred file and directory addresses

for caching purposes. It is necessary to edit this file before running the GekkoFS daemon.

```txt

\lustre_root

\gekko_root

\lustre_root\file1

\lustre_root\dir1

```

#### Interception library

Lustre HSM integration receives a system call and, based on that, dispatches the system call between Lustre and GekkoFS.

It is necessary to load the interception library before running anything else.

```bash

    export LIBGKFS_HOSTS_FILE=<hostfile_path>

```

#### Staging library

The staging library provides essential functions for managing transfer operations and controlling connections with the

HSM. This library is integrated into the building process alongside the daemon and interception library. The daemon and

interception layer utilize this library for prefetching, flushing data, and updating data status on the HSM.

#### Running the GekkoFS daemon

The HSM extension is integrated into the GekkoFS daemon. Users only need to run one daemon on each node. This daemon

runs the copy manager, which is responsible for receiving and sending requests to Lustre HSM and is packaged alongside

the GekkoFS binary. This integrated daemon is tested with sockets and verbs.

**This command requires administrative permissions to run.**

```bash

  mpirun -np <N> <install_path>/bin/gkfs_daemon -r <fs_data_path> -m <pseudo_gkfs_mount_dir_path>\

   -H <hostsfile_path> -P <ofi+sockets\ofi+verbs>

```

#### Running application

For running any application, it is essential to pass the hostsfile and interception library to application.

```bash

  mpirun -np <N> -x LIBGKFS_HOSTS_FILE="$LIBGKFS_HOSTS_FILE" -x LD_PRELOAD="$LIBGKFS_INTERCEPT" "$APP_BIN"

```

# Miscellaneous

## External functions

//path_a

//path_b

    preload_context.hpp

    preload_util.hpp

    cache.hpp

    direct_created.hpp

    find_path.hpp

    removed_path.hpp

    rpc/rpc_types.hpp

    rpc/forward_management.hpp

    rpc/forward_metadata.hpp

    preload_context.hpp

    preload_util.hpp

    cache.hpp

    direct_created.hpp

    find_path.hpp

    removed_path.hpp

    rpc/rpc_types.hpp

    rpc/forward_management.hpp

    rpc/forward_metadata.hpp