Commit 77666c04 authored by Marc Vef's avatar Marc Vef
Browse files

Merge branch '179-bump-all-dependencies' into 'master'

Resolve "Bump all dependencies"

Closes #179, #145, and #46

See merge request !112
parents fc8d4029 8b1dd0ea
Pipeline #2257 passed with stages
in 22 minutes and 45 seconds
......@@ -22,14 +22,14 @@ variables:
GIT_SUBMODULE_STRATEGY: recursive
# base image
image: gekkofs/core:0.8.0
image: gekkofs/core:0.9.0
################################################################################
## Validating
################################################################################
check format:
stage: lint
image: gekkofs/linter:0.8.0
image: gekkofs/linter:0.9.0
needs: []
script:
- ${SCRIPTS_DIR}/check_format.sh
......@@ -43,7 +43,7 @@ check format:
################################################################################
gkfs:
stage: build
image: gekkofs/deps:0.8.0
image: gekkofs/deps:0.9.0
interruptible: true
needs: []
script:
......@@ -56,6 +56,7 @@ gkfs:
-DGKFS_BUILD_TESTS:BOOL=ON
-DGKFS_INSTALL_TESTS:BOOL=ON
-DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}
-DGKFS_USE_GUIDED_DISTRIBUTION:BOOL=ON
${CI_PROJECT_DIR}
- make -j$(nproc) install
# reduce artifacts size
......@@ -68,7 +69,7 @@ gkfs:
gkfwd:
stage: build
image: gekkofs/deps:0.8.0
image: gekkofs/deps:0.9.0
interruptible: true
needs: []
script:
......@@ -101,7 +102,7 @@ gkfwd:
## == tests for scripts ====================
scripts:
stage: test
image: gekkofs/testing:0.8.0
image: gekkofs/testing:0.9.0
needs: []
parallel:
matrix:
......@@ -119,7 +120,7 @@ scripts:
## == integration tests for gkfs ===========
gkfs:integration:
stage: test
image: gekkofs/testing:0.8.0
image: gekkofs/testing:0.9.0
interruptible: true
needs: ['gkfs']
parallel:
......@@ -164,7 +165,7 @@ gkfs:integration:
## == integration tests for gkfwd ==========
gkfwd:integration:
stage: test
image: gekkofs/testing:0.8.0
image: gekkofs/testing:0.9.0
interruptible: true
needs: ['gkfwd']
parallel:
......@@ -209,7 +210,7 @@ gkfwd:integration:
## == unit tests for gkfs ==================
gkfs:unit:
stage: test
image: gekkofs/testing:0.8.0
image: gekkofs/testing:0.9.0
needs: ['gkfs']
script:
## run actual tests
......@@ -237,7 +238,7 @@ gkfs:unit:
################################################################################
coverage:
stage: report
image: gekkofs/coverage:0.8.0
image: gekkofs/coverage:0.9.0
needs: [ 'gkfs:integration', 'gkfwd:integration', 'gkfs:unit' ]
script:
- cd ${BUILD_PATH}
......
......@@ -16,3 +16,6 @@
[submodule "tests/scripts/helpers/bats-file"]
path = tests/scripts/helpers/bats-file
url = https://github.com/bats-core/bats-file.git
[submodule "external/spdlog"]
path = external/spdlog
url = https://github.com/gabime/spdlog
################################################################################
# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
# #
# This software was partially supported by the #
# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
# #
# This software was partially supported by the #
# ADA-FS project under the SPPEXA project funded by the DFG. #
# #
# This file is part of GekkoFS. #
# #
# GekkoFS is free software: you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation, either version 3 of the License, or #
# (at your option) any later version. #
# #
# GekkoFS is distributed in the hope that it will be useful, #
# but WITHOUT ANY WARRANTY; without even the implied warranty of #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
# GNU General Public License for more details. #
# #
# You should have received a copy of the GNU General Public License #
# along with GekkoFS. If not, see <https://www.gnu.org/licenses/>. #
# #
# SPDX-License-Identifier: GPL-3.0-or-later #
################################################################################
include(CMakeParseArguments)
#[=======================================================================[.rst:
get_cmake_variables(OUTPUT_VARIABLE
[ REGEX <regular_expression> [EXCLUDE] ])
Initialize ``OUTPUT_VARIABLE`` to a list of all currently defined CMake
variables. The function accepts a ``<regular_expression>`` to allow filtering
the results. Furthermore, if the ``EXCLUDE`` flag is used, the function will
return all variables NOT MATCHING the provided ``<regular_expression>``.
#]=======================================================================]
function(get_cmake_variables OUTPUT_VARIABLE)
set(OPTIONS EXCLUDE)
set(SINGLE_VALUE REGEX)
set(MULTI_VALUE) # no multiple value args for now
cmake_parse_arguments(
ARGS "${OPTIONS}" "${SINGLE_VALUE}" "${MULTI_VALUE}" ${ARGN}
)
if(ARGS_UNPARSED_ARGUMENTS)
message(WARNING "Unparsed arguments in get_cmake_variables(): "
"this often indicates typos!\n"
"Unparsed arguments: ${ARGS_UNPARSED_ARGUMENTS}"
)
endif()
get_cmake_property(_var_names VARIABLES)
if(NOT ARGS_REGEX)
set(${OUTPUT_VARIABLE}
${_var_names}
PARENT_SCOPE
)
return()
endif()
if(ARGS_EXCLUDE)
set(_mode EXCLUDE)
else()
set(_mode INCLUDE)
endif()
list(FILTER _var_names ${_mode} REGEX ${ARGS_REGEX})
set(${OUTPUT_VARIABLE}
${_var_names}
PARENT_SCOPE
)
endfunction()
#[=======================================================================[.rst:
dump_cmake_variables([ REGEX <regular_expression> [EXCLUDE] ])
Print all currently defined CMake variables. The function accepts a
``<regular_expression>`` to allow filtering the results. Furthermore, if the
``EXCLUDE`` flag is used, the function will print all variables NOT MATCHING
the provided ``<regular_expression>``.
#]=======================================================================]
function(dump_cmake_variables)
set(OPTIONS EXCLUDE)
set(SINGLE_VALUE REGEX)
set(MULTI_VALUE) # no multiple value args for now
cmake_parse_arguments(
ARGS "${OPTIONS}" "${SINGLE_VALUE}" "${MULTI_VALUE}" ${ARGN}
)
if(ARGS_UNPARSED_ARGUMENTS)
message(WARNING "Unparsed arguments in dump_cmake_variables(): "
"this often indicates typos!"
"Unparsed arguments: ${ARGS_UNPARSED_ARGUMENTS}"
)
endif()
if(ARGS_EXCLUDE AND NOT ARGS_REGEX)
message(ERROR "EXCLUDE option doesn't make sense without REGEX.")
endif()
get_cmake_variables(_var_names REGEX ${ARGS_REGEX} ${ARGS_EXCLUDE})
foreach(_var ${_var_names})
message(STATUS "${_var}=${${_var}}")
endforeach()
endfunction()
#[=======================================================================[.rst:
mark_variables_as_advanced(REGEX <regular_expression>)
Mark all CMake variables matching ``regular_expression`` as advanced.
#]=======================================================================]
function(mark_variables_as_advanced)
set(OPTIONS) # no options for now
set(SINGLE_VALUE REGEX)
set(MULTI_VALUE) # no multiple value args for now
cmake_parse_arguments(
ARGS "${OPTIONS}" "${SINGLE_VALUE}" "${MULTI_VALUE}" ${ARGN}
)
if(ARGS_UNPARSED_ARGUMENTS)
message(WARNING "Unparsed arguments in mark_variables_as_advanced(): "
"this often indicates typos!\n"
"Unparsed arguments: ${ARGS_UNPARSED_ARGUMENTS}"
)
endif()
get_cmake_property(_var_names VARIABLES)
list(FILTER _var_names INCLUDE REGEX ${ARGS_REGEX})
foreach(_var ${_var_names})
mark_as_advanced(${_var})
endforeach()
endfunction()
......@@ -65,6 +65,11 @@ set(CMAKE_C_FLAGS_MEMCHECK "${WARNINGS_FLAGS} -g -O0 -fsanitize=address -fno-omi
set(CMAKE_C_FLAGS_MAINTAINER "${WARNINGS_FLAGS} -g -O0 -pg -no-pie")
mark_as_advanced(CMAKE_CXX_FLAGS_MAINTAINER)
# CMake and general includes
include(CheckCXXSourceCompiles)
include(CMakeDependentOption)
include(GNUInstallDirs)
# Project version
set(GIT_VERSION_FOUND FALSE)
if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/.git)
......@@ -116,10 +121,6 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS 0)
# Rocksdb dependencies
find_package(LZ4 REQUIRED)
find_package(ZLIB REQUIRED)
find_package(BZip2 REQUIRED)
find_package(Snappy REQUIRED)
find_package(ZStd REQUIRED)
find_package(JeMalloc) # required if rocksdb has been build with jemalloc
find_package(RocksDB REQUIRED)
# margo dependencies
......@@ -127,8 +128,6 @@ find_package(Mercury REQUIRED)
find_package(Abt REQUIRED)
find_package(Margo REQUIRED)
find_package(Syscall_intercept REQUIRED)
find_package(Threads REQUIRED)
# some compilers need extra flags for std::filesystem, such as -lstdc++fs, this
......@@ -138,6 +137,9 @@ find_package(Filesystem REQUIRED)
find_package(Date REQUIRED)
# Import some convenience functions
include(gkfs-utils)
option(CREATE_CHECK_PARENTS "Check parent directory existance before creating child node" ON)
message(STATUS "[gekkofs] Create checks parents: ${CREATE_CHECK_PARENTS}")
......@@ -181,9 +183,11 @@ mark_as_advanced(CLIENT_LOG_MESSAGE_SIZE)
option(GKFS_USE_GUIDED_DISTRIBUTION "Use guided data distributor " OFF)
message(STATUS "[gekkofs] Guided data distributor: ${GKFS_USE_GUIDED_DISTRIBUTION}")
set(GKFS_USE_GUIDED_DISTRIBUTION_PATH "/tmp/guided.txt" CACHE STRING "File Path for guided distributor")
set_property(CACHE GKFS_USE_GUIDED_DISTRIBUTION_PATH PROPERTY STRINGS)
message(STATUS "[gekkofs] Guided data distributor input file path: ${GKFS_USE_GUIDED_DISTRIBUTION_PATH}")
if(GKFS_USE_GUIDED_DISTRIBUTION)
set(GKFS_USE_GUIDED_DISTRIBUTION_PATH "/tmp/guided.txt" CACHE STRING "File Path for guided distributor")
set_property(CACHE GKFS_USE_GUIDED_DISTRIBUTION_PATH PROPERTY STRINGS)
message(STATUS "[gekkofs] Guided data distributor input file path: ${GKFS_USE_GUIDED_DISTRIBUTION_PATH}")
endif()
configure_file(include/common/cmake_configure.hpp.in include/common/cmake_configure.hpp)
......@@ -193,10 +197,6 @@ target_link_libraries(RocksDB
INTERFACE
${ROCKSDB_LIBRARIES}
# rocksdb libs
${Snappy_LIBRARIES}
${ZLIB_LIBRARIES}
${BZIP2_LIBRARIES}
${ZStd_LIBRARIES}
${LZ4_LIBRARIES}
)
......@@ -212,12 +212,6 @@ set_target_properties(RocksDB
INTERFACE_INCLUDE_DIRECTORIES ${ROCKSDB_INCLUDE_DIRS}
)
add_library(spdlog INTERFACE)
# we cannot use target_include_directories with CMake < 3.11
set_target_properties(spdlog
PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/external"
)
add_library(CLI11 INTERFACE)
# we cannot use target_include_directories with CMake < 3.11
set_target_properties(CLI11
......@@ -246,8 +240,6 @@ include_directories(
${CMAKE_BINARY_DIR}/include
)
include(GNUInstallDirs)
# Common components
add_subdirectory(src/common)
# Daemon
......@@ -255,9 +247,13 @@ add_subdirectory(src/daemon)
# Client library
add_subdirectory(src/client)
### Mark any CMake variables imported from {fmt} and spdlog as advanced, so
### that they don't appear in cmake-gui or ccmake. Similarly for FETCHCONTENT
### variables.
mark_variables_as_advanced(REGEX "^(FETCHCONTENT|fmt|FMT|spdlog|SPDLOG)_.*$")
option(GKFS_BUILD_TESTS "Build GekkoFS self tests" OFF)
include(CMakeDependentOption)
cmake_dependent_option(GKFS_INSTALL_TESTS "Install GekkoFS self tests" OFF "GKFS_BUILD_TESTS" OFF)
if (GKFS_BUILD_TESTS)
......@@ -289,6 +285,13 @@ if (GKFS_BUILD_TESTS)
set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
endif ()
message(STATUS "[gekkofs] Forwarding tests: ${GKFS_TESTS_FORWARDING}")
message(STATUS "[gekkofs] Check for guided distributor tests...")
if (GKFS_USE_GUIDED_DISTRIBUTION)
set(GKFS_TESTS_GUIDED_DISTRIBUTION "ON" CACHE STRING "Enable guided distributor tests (default: OFF)")
else()
set(GKFS_TESTS_GUIDED_DISTRIBUTION "OFF" CACHE STRING "Enable guided distributor tests (default: OFF)")
endif()
message(STATUS "[gekkofs] Guided distributor tests: ${GKFS_TESTS_GUIDED_DISTRIBUTION}")
add_subdirectory(tests)
add_subdirectory(examples/gfind)
......
......@@ -11,136 +11,46 @@ to I/O, which reduces interferences and improves performance.
# Dependencies
- Upgrade your gcc to version at least 4.8 to get C++11 support
- CMake >3.6 (>3.11 for GekkoFS testing)
- \>gcc-8 (including g++) for C++11 support
- General build tools: Git, Curl, CMake >3.6 (>3.11 for GekkoFS testing), Autoconf, Automake
- Miscellaneous: Libtool, Libconfig
## Debian/Ubuntu - Dependencies
### Debian/Ubuntu
GekkoFS base dependencies: `apt install git curl cmake autoconf automake libtool libconfig-dev`
- snappy: `sudo apt-get install libsnappy-dev`
- zlib: `sudo apt-get install zlib1g-dev`
- bzip2: `sudo apt-get install libbz2-dev`
- zstandard: `sudo apt-get install libzstd-dev`
- lz4: `sudo apt-get install liblz4-dev`
- uuid: `sudo apt-get install uuid-dev`
- capstone: `sudo apt-get install libcapstone-dev`
GekkoFS testing support: `apt install python3-dev python3 python3-venv`
### CentOS/Red Hat - Dependencies
With testing
### CentOS/Red Hat
GekkoFS base dependencies: `yum install gcc-c++ git curl cmake autoconf automake libtool libconfig`
GekkoFS testing support: `python38-devel` (**>Python-3.6 required**)
- snappy: `sudo yum install snappy snappy-devel`
- zlib: `sudo yum install zlib zlib-devel`
- bzip2: `sudo yum install bzip2 bzip2-devel`
- zstandard:
```bash
wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
tar zxvf zstd-1.1.3.tar.gz
cd zstd-1.1.3
make && sudo make install
```
- lz4: `sudo yum install lz4 lz4-devel`
- uuid: `sudo yum install libuuid-devel`
- capstone: `sudo yum install capstone capstone-devel`
# Usage
## Clone and compile direct GekkoFS dependencies
- Go to the `scripts` folder and first clone all dependencies projects. You can choose the according network (na) plugin
(execute the script for help):
```bash
usage: dl_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CONFIG>] [-d <DEPENDENCY>]
source_path
This script gets all GekkoFS dependency sources (excluding the fs itself)
positional arguments:
source_path path where the dependency downloads are put
optional arguments:
-h, --help shows this help message and exits
-l, --list-dependencies
list dependencies available for download with descriptions
-n <NAPLUGIN>, --na <NAPLUGIN>
network layer that is used for communication. Valid: {bmi,ofi,all}
defaults to 'ofi'
-c <CONFIG>, --config <CONFIG>
allows additional configurations, e.g., for specific clusters
supported values: {mogon2, mogon1, ngio, direct, all}
defaults to 'direct'
-d <DEPENDENCY>, --dependency <DEPENDENCY>
download a specific dependency and ignore --config setting. If unspecified
all dependencies are built and installed based on set --config setting.
Multiple dependencies are supported: Pass a space-separated string (e.g., "ofi mercury"
-v, --verbose Increase download verbosity
```
- Now use the install script to compile them and install them to the desired directory. You can choose the according
(na) network plugin (execute the script for help):
# Step-by-step installation
```bash
usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CONFIG>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>]
source_path install_path
This script compiles all GekkoFS dependencies (excluding the fs itself)
positional arguments:
source_path path to the cloned dependencies path from clone_dep.sh
install_path path to the install path of the compiled dependencies
optional arguments:
-h, --help shows this help message and exits
-l, --list-dependencies
list dependencies available for building and installation
-n <NAPLUGIN>, --na <NAPLUGIN>
network layer that is used for communication. Valid: {bmi,ofi,all}
defaults to 'all'
-c <CONFIG>, --config <CONFIG>
allows additional configurations, e.g., for specific clusters
supported values: {mogon1, mogon2, ngio, direct, all}
defaults to 'direct'
-d <DEPENDENCY>, --dependency <DEPENDENCY>
download a specific dependency and ignore --config setting. If unspecified
all dependencies are built and installed based on set --config setting.
Multiple dependencies are supported: Pass a space-separated string (e.g., "ofi mercury"
-j <COMPILE_CORES>, --compilecores <COMPILE_CORES>
number of cores that are used to compile the dependencies
defaults to number of available cores
-t, --test Perform libraries tests.
```
## Compile GekkoFS
1. Make sure the above listed dependencies are available on your machine
2. Clone GekkoFS: `git clone --recurse-submodules https://storage.bsc.es/gitlab/hpc/gekkofs.git`
3. Set up the necessary environment variables where the compiled direct GekkoFS dependencies will be installed at (we assume the path `/home/foo/gekkofs_deps/install` in the following)
- `export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/home/foo/gekkofs_deps/install/lib:/home/foo/gekkofs_deps/install/lib64`
4. Download and compile the direct dependencies, e.g.,
- Download example: `gekkofs/scripts/dl_dep.sh /home/foo/gekkofs_deps/git`
- Compilation example: `gekkofs/scripts/compile_dep.sh /home/foo/gekkofs_deps/git /home/foo/gekkofs_deps/install`
- Consult `-h` for additional arguments for each script
5. Compile GekkoFS and run optional tests
- Create build directory: `mkdir gekkofs/build && cd gekkofs/build`
- Configure GekkoFS: `cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=/home/foo/gekkofs_deps/install ..`
- add `-DCMAKE_INSTALL_PREFIX=<install_path>` where the GekkoFS client library and server executable should be available
- add `-DGKFS_BUILD_TESTS=ON` if tests should be build
- Build and install GekkoFS: `make -j8 install`
- Run tests: `make test`
If above dependencies have been installed outside of the usual system paths, use CMake's `-DCMAKE_PREFIX_PATH` to
make this path known to CMake.
```bash
mkdir build && cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
make
```
In order to build self-tests, the *optional* GKFS_BUILD_TESTS CMake option needs
to be enabled when building. Once that is done, tests can be run by running
`make test` in the `build` directory:
```bash
mkdir build && cd build
cmake -DGKFS_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Release ..
make
make test
make install
```
GekkoFS is now available at:
- GekkoFS daemon (server): `<install_path>/bin/gkfs_daemon`
- GekkoFS client interception library: `<install_path>/lib64/libgkfs_intercept.so`
**IMPORTANT:** Please note that the testing framework requires Python 3.6 and >CMake 3.11 as
additional dependencies in order to run.
# Run GekkoFS
## Run GekkoFS
## General
On each node a daemon (`gkfs_daemon` binary) has to be started. Other tools can be used to execute
the binary on many nodes, e.g., `srun`, `mpiexec/mpirun`, `pdsh`, or `pssh`.
......@@ -152,51 +62,64 @@ The `-P` argument is used for setting another RPC protocol. See below.
- `ofi+tcp` for using the libfabric plugin with TCP (slower than sockets)
- `ofi+verbs` for using the libfabric plugin with Infiniband verbs (reasonably stable)
- `ofi+psm2` for using the libfabric plugin with Intel Omni-Path (unstable)
- `bmi+tcp` for using the bmi plugin with TCP (alternative to libfabric)
### Start and shut down daemon directly
## The GekkoFS hostsfile
Each GekkoFS daemon needs to register itself in a shared file (*hostsfile*) which needs to be accessible to _all_ GekkoFS clients and daemons.
Therefore, the hostsfile describes a file system and which node is part of that specific GekkoFS file system instance.
In a typical cluster environment this hostsfile should be placed within a POSIX-compliant parallel file system, such as GPFS or Lustre.
*Note: NFS is not strongly consistent and cannot be used for the hosts file!*
## GekkoFS daemon start and shut down
tl;dr example: `<install_path>/bin/gkfs_daemon -r <fs_data_path> -m <pseudo_gkfs_mount_dir_path> -H <hostsfile_path>`
`./build/src/daemon/gkfs_daemon -r <fs_data_path> -m <pseudo_mount_dir_path>`
Run the GekkoFS daemon on each node specifying its locally used directory where the file system data and metadata is stored (`-r/--rootdir <fs_data_path>`), e.g., the node-local SSD;
2. the pseudo mount directory used by clients to access GekkoFS (`-m/--mountdir <pseudo_gkfs_mount_dir_path>`); and
3. the hostsfile path (`-H/--hostsfile <hostfile_path>`).
Further options are available:
Further options:
```bash
Allowed options:
-h [ --help ] Help message
-m [ --mountdir ] arg Virtual mounting directory where GekkoFS is
available.
-r [ --rootdir ] arg Local data directory where GekkoFS data for this
daemon is stored.
-i [ --metadir ] arg Metadata directory where GekkoFS RocksDB data
directory is located. If not set, rootdir is used.
-l [ --listen ] arg Address or interface to bind the daemon to.
Default: local hostname.
When used with ofi+verbs the FI_VERBS_IFACE
environment variable is set accordingly which
associates the verbs device with the network
interface. In case FI_VERBS_IFACE is already
defined, the argument is ignored. Default 'ib'.
-H [ --hosts-file ] arg Shared file used by deamons to register their
endpoints. (default './gkfs_hosts.txt')
-P [ --rpc-protocol ] arg Used RPC protocol for inter-node communication.
Available: {ofi+sockets, ofi+verbs, ofi+psm2} for
TCP, Infiniband, and Omni-Path, respectively.
(Default ofi+sockets)
Libfabric must have enabled support verbs or psm2.
--auto-sm Enables intra-node communication (IPCs) via the
`na+sm` (shared memory) protocol, instead of using
the RPC protocol. (Default off)
--version Print version and exit.
Allowed options
Usage: bin/gkfs_daemon [OPTIONS]
Options:
-h,--help Print this help message and exit
-m,--mountdir TEXT REQUIRED Virtual mounting directory where GekkoFS is available.
-r,--rootdir TEXT REQUIRED Local data directory where GekkoFS data for this daemon is stored.
-i,--metadir TEXT Metadata directory where GekkoFS RocksDB data directory is located. If not set, rootdir is used.
-l,--listen TEXT Address or interface to bind the daemon to. Default: local hostname.
When used with ofi+verbs the FI_VERBS_IFACE environment variable is set accordingly which associates the verbs device with the network interface. In case FI_VERBS_IFACE is already defined, the argument is ignored. Default 'ib'.
-H,--hosts-file TEXT Shared file used by deamons to register their endpoints. (default './gkfs_hosts.txt')
-P,--rpc-protocol TEXT Used RPC protocol for inter-node communication.
Available: {ofi+sockets, ofi+verbs, ofi+psm2} for TCP, Infiniband, and Omni-Path, respectively. (Default ofi+sockets)
Libfabric must have enabled support verbs or psm2.
--auto-sm Enables intra-node communication (IPCs) via the `na+sm` (shared memory) protocol, instead of using the RPC protocol. (Default off)
--clean-rootdir Cleans Rootdir >before< launching the deamon
--version Print version and exit.
```
Shut it down by gracefully killing the process (SIGTERM).
## Miscellaneous
## Use the GekkoFS client library
tl;dr example:
```bash
export LIBGKFS_ HOSTS_FILE=<hostfile_path>
LD_PRELOAD=<install_path>/lib64/libgkfs_intercept.so cp ~/some_input_data <pseudo_gkfs_mount_dir_path>/some_input_data
LD_PRELOAD=<install_path>/lib64/libgkfs_intercept.so md5sum ~/some_input_data <pseudo_gkfs_mount_dir_path>/some_input_data
```
Clients read the hostsfile to determine which daemons are part of the GekkoFS instance.
Because the client is an interposition library that is loaded within the context of the application, this information is passed via the environment variable `LIBGKFS_HOSTS_FILE` pointing to the hostsfile path.
The client library itself is loaded for each application process via the `LD_PRELOAD` environment variable intercepting file system related calls.
If they are within (or hierarchically under) the GekkoFS mount directory they are processed in the library, otherwise they are passed to the kernel.
Metadata and actual data will be stored at the `<fs_data_path>`. The path where the application works on is set with
`<pseudo_mount_dir_path>`.
Note, if `LD_PRELOAD` is not pointing to the library and, hence the client is not loaded, the mounting directory appear to be empty.
Run the application with the preload library: `LD_PRELOAD=<path>/build/lib/libgkfs_intercept.so ./application`. In the case of
an MPI application use the `{mpirun, mpiexec} -x` argument.
For MPI application, the `LD_PRELOAD` variable can be passed with the `-x` argument for `mpirun/mpiexec`.
### Logging
The following environment variables can be used to enable logging in the client
......@@ -245,50 +168,60 @@ can be provided to set the path to the log file, and the log module can be
selected with the `GKFS_LOG_LEVEL={off,critical,err,warn,info,debug,trace}`
environment variable.
# Miscellaneous
### External functions
## External functions
GekkoFS allows to use external functions on your client code, via LD_PRELOAD.
Source code needs to be compiled with -fPIC. We include a pfind io500 substitution,
`examples/gfind/gfind.cpp` and a non-mpi version `examples/gfind/sfind.cpp`
### Data distributors
## Data distributors
The data distribution can be selected at compilation time, we have 2 distributors available: