Commit 7db4b597 authored by Ramon Nou's avatar Ramon Nou
Browse files

Merge branch 'sfind' into 'master' 

directory optimization with compression and reattemp

* Refactor sfind so it can use SLURM\_ environment variables to ask to different servers.
* Create a sample bash script to gather all the info (map-\>reduce)
* Compress directory data with zstd.
* Make a new config.hpp option for controlling the compression
* If directory buffer is not enough it will reattempt with the exact size
* Issues related : #372 , #371 , #370

See merge request !270
parents 1167c20e 2510eb20

CHANGELOG.md

+13 −0
Original line number Diff line number Diff line
@@ -5,6 +5,19 @@ All notable changes to GekkoFS project will be documented in this file. 
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres

to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).



## [Unreleased]

### New

  - directory optimization with compression and reattemp ([!270](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/270))

    - Refactor sfind so it can use SLURM_ environment variables to ask to different servers.

    - Create a sample bash script to gather all the info (map->reduce)

    - Compress directory data with zstd.

    - Make a new config.hpp option for controlling the compression

    - If directory buffer is not enough it will reattempt with the exact size



### Changed 



### Fixed



## [0.9.5] - 2025-08

### New

  - Added cppcheck code checking capabilities ([!214](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/214))

CMake/FindZStd.cmake

+38 −21
Original line number Diff line number Diff line
@@ -26,32 +26,49 @@ 
# SPDX-License-Identifier: GPL-3.0-or-later                                    #

################################################################################



#

# - Try to find Facebook zstd library

# This will define

# ZStd_FOUND

# ZStd_INCLUDE_DIR

# ZStd_LIBRARIES

#





# Standard names to search for

set(ZStd_NAMES zstd zstd_static)



find_path(ZStd_INCLUDE_DIR

          NAMES zstd.h

)

          PATH_SUFFIXES include)



find_library(ZStd_LIBRARY

    NAMES zstd

)

# Allow ZStd_LIBRARY to be set manually, as the location of the zstd library

if(NOT ZStd_LIBRARY)

  find_library(ZStd_LIBRARY_RELEASE

               NAMES ${ZStd_NAMES}

               PATH_SUFFIXES lib)

  

set(ZStd_LIBRARIES ${ZStd_LIBRARY})

set(ZStd_INCLUDE_DIRS ${ZStd_INCLUDE_DIR})



  include(SelectLibraryConfigurations)

  select_library_configurations(ZStd)

endif()



unset(ZStd_NAMES)



mark_as_advanced(ZStd_INCLUDE_DIR)



include(FindPackageHandleStandardArgs)

FIND_PACKAGE_HANDLE_STANDARD_ARGS(ZStd

        REQUIRED_VARS ZStd_LIBRARY ZStd_INCLUDE_DIR

        VERSION_VAR ZStd_VERSION_STRING)



if(ZStd_FOUND)

    set(ZStd_INCLUDE_DIRS ${ZStd_INCLUDE_DIR})



    if(NOT ZStd_LIBRARIES)

        set(ZStd_LIBRARIES ${ZStd_LIBRARY})

    endif()



    if(NOT TARGET ZStd::ZStd)

        add_library(ZStd::ZStd UNKNOWN IMPORTED)

        set_target_properties(ZStd::ZStd PROPERTIES

                INTERFACE_INCLUDE_DIRECTORIES "${ZStd_INCLUDE_DIRS}")

       

find_package_handle_standard_args(ZStd

    DEFAULT_MSG ZStd_LIBRARY ZStd_INCLUDE_DIR

)

        set_target_properties(ZStd::ZStd PROPERTIES

                IMPORTED_LOCATION "${ZStd_LIBRARY}")



mark_as_advanced(

    ZStd_LIBRARY

    ZStd_INCLUDE_DIR

)
 
 No newline at end of file
 
    endif()

endif()

CMakeLists.txt

+3 −0
Original line number Diff line number Diff line
@@ -161,6 +161,9 @@ find_package(Margo 0.14.0 REQUIRED) 
message(STATUS "[${PROJECT_NAME}] Checking for syscall_intercept")

find_package(Syscall_intercept REQUIRED)



message(STATUS "[${PROJECT_NAME}] Checking for Zstd")

find_package(ZStd REQUIRED)



### AGIOS: required for scheduling I/O requests

if (GKFS_ENABLE_AGIOS)

    message(STATUS "[${PROJECT_NAME}] Checking for Agios")

README.md

+3 −0
Original line number Diff line number Diff line
@@ -587,6 +587,9 @@ Client-metrics require the CMake argument `-DGKFS_ENABLE_CLIENT_METRICS=ON` (see 
- `LIBGKFS_METRICS_IP_PORT` - Enable flushing to a set ZeroMQ server (replaces `LIBGKFS_METRICS_PATH`).

- `LIBGKFS_PROXY_PID_FILE` - Path to the proxy pid file (when using the GekkoFS proxy).

- `LIBGKFS_NUM_REPL` - Number of replicas for data.

#### Directory optimizations

Set `true` the variable `use_dirents_compression` available at `include/config.hpp` to transfer directories compressed with zstd.



#### Caching

##### Dentry cache

Improves performance for `ls -l` type operations by caching file metadata for subsequent `stat()` operations during

examples/gfind/CMakeLists.txt

+1 −0
Original line number Diff line number Diff line
@@ -29,6 +29,7 @@ 
set (CMAKE_CXX_STANDARD 17)



add_executable(sfind sfind.cpp)

target_link_libraries(sfind PRIVATE ZStd::ZStd)

set_property(TARGET sfind PROPERTY POSITION_INDEPENDENT_CODE ON)

if(GKFS_INSTALL_TESTS)

    install(TARGETS sfind