Commit d97afd3d authored by Ramon Nou's avatar Ramon Nou
Browse files

Merge branch 'rnou/40-interface-with-ftio' into 'main' 

Resolve "Interface with FTIO"

Closes #40

See merge request !30
parents a362795c f62de23b

.gitignore

+2 −2
Original line number Diff line number Diff line 
build*

cmake-build-debug

cmake-build-*

.idea

.vscode
 
 No newline at end of file

CMakeLists.txt

+12 −1
Original line number Diff line number Diff line
@@ -30,7 +30,7 @@ cmake_minimum_required(VERSION 3.19) 


project(

  cargo

  VERSION 0.3.5

  VERSION 0.3.6

  LANGUAGES C CXX

)
@@ -281,6 +281,17 @@ if (Expand_FOUND) 
  message(STATUS "[${PROJECT_NAME}] Found Expand")

endif()



### DataClay: Optional for DataClay

find_package(DataClay)

if (DataClay_FOUND)

  add_compile_definitions(DATACLAY_PLUGIN)

  add_compile_definitions(DataClay_PATH="${DataClay_MODEL_DIR}/dataclay-plugin")

  message(STATUS "[${PROJECT_NAME}] Found DataClay")

endif()









### Threads: required by ASIO

find_package(Threads REQUIRED)

COPYRIGHT_NOTICE

+1 −1
Original line number Diff line number Diff line 
/******************************************************************************

 * Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain

 * Copyright 2022-2024, Barcelona Supercomputing Center (BSC), Spain

 *

 * This software was partially supported by the EuroHPC-funded project ADMIRE

 *   (Project ID: 956748, https://www.admire-eurohpc.eu).

README.md

+19 −2
Original line number Diff line number Diff line
@@ -190,8 +190,25 @@ cli/ccp --server ofi+tcp://127.0.0.1:62000 --input /directory/subdir --output /d 
`--if or --of` can be: posix, gekkofs, hercules, dataclay, expand and parallel (for MPIIO requests, but only one side is allowed).



Typically you should use posix or parallel and then one specialized adhocfs. Posix is also able to be used with LD_PRELOAD, however

higher performance and flexibility can be obtained using the specific configuration.

higher performance and flexibility can be obtained using the specific configuration. Some backends are only available with directory support for stage-in. 



On the other hand, MPIIO (parallel) uses normally file locking so there is a performance imapact, and posix is faster (we supose no external modifications are done).



Other commands are `ping`, `shutdown` and `shaping` (for bw control).
 
 No newline at end of file
 
Other commands are `ping`, `shutdown`, `shaping` (for bw control) and `cargo_ftio` to interactions with ftio (stage-out and gekkofs)



`cargo_ftio` provides --resume, --pause and --run options to pause and resume the ftio related transfers. We set ftio transfers, the transfers that have gekkofs as --of, that had been setup after a ftio command.



```shell

#SETUP FTIO, this enables stage-out to be delayed (10000 seconds)

cargo_ftio --server tcp://127.0.0.1:62000 -c -1 -p -1 -t 10000

#SETUP Stage-out (monitors data directory and subdirs for new file)

ccp --server tcp://127.0.0.1:62000 --input /data --output ~/stage-out --if gekkofs --of parallel

#UPDATE FTIO (as needed, each 25 seconds will do the transfer order)

cargo_ftio --server tcp://127.0.0.1:62000 -c -1 -p -1 -t 25

```



## User libraries for adhocfs

If Cargo finds the adhoc fs libraries (we support GekkoFS and dataclay, in this release), it will automatically use them.

The CMake command will show which adhocfs are detected.



On the other hand, LD_preload techniques could be used.
 
 No newline at end of file

cli/CMakeLists.txt

+20 −1
Original line number Diff line number Diff line
@@ -101,7 +101,26 @@ target_link_libraries(shaping 
    cargo

)



install(TARGETS cargo_ping cargo_shutdown ccp shaping



################################################################################

## ftio: A CLI tool to send the ftio info to a Cargo server 

add_executable(cargo_ftio)



target_sources(cargo_ftio

  PRIVATE

    ftio.cpp

)



target_link_libraries(cargo_ftio

  PUBLIC

    fmt::fmt

    CLI11::CLI11

    net::rpc_client

    cargo

)





install(TARGETS cargo_ping cargo_shutdown ccp shaping cargo_ftio

        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}

)