Commit 4e49ba36 authored by Ramon Nou's avatar Ramon Nou
Browse files

Merge branch 'master' into 'rnou/379-add-compression-on-demand-on-data'

# Conflicts:
#   .gitlab-ci.yml
#   src/client/gkfs_data.cpp
#   src/client/preload.cpp
parents c4d06d0d 990a3560
Loading
Loading
Loading
Loading
Loading
+95 −33
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ stages:
  - deploy

variables:
  VERSION:                      "0.9.6"
  SCRIPTS_DIR:                  "${CI_PROJECT_DIR}/scripts"
  CI_SCRIPTS_DIR:               "${CI_PROJECT_DIR}/scripts/ci"
  BUILD_PATH:                   "${CI_PROJECT_DIR}/gkfs/build"
@@ -23,8 +24,17 @@ variables:
  LIBGKFS_LOG_OUTPUT:           "${CI_PROJECT_DIR}/logs/gkfs_client.log"
  GIT_SUBMODULE_STRATEGY:       recursive
  CCACHE_DIR:                   "${CI_PROJECT_DIR}/ccache"
#  SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar"  # Defines the location of the analysis task cache
#  GIT_DEPTH: "0"  # Tells git to fetch all the branches of the project, required by the analysis task

  IMAGE_PREFIX:                "${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/"
  #IMAGE_PREFIX:                "gekkofs/"

  CORE:                        "${IMAGE_PREFIX}core:${VERSION}"
  LINTER:                      "${IMAGE_PREFIX}linter:${VERSION}"
  DEPS:                        "${IMAGE_PREFIX}deps:${VERSION}"
  TESTING:                     "${IMAGE_PREFIX}testing:${VERSION}"
  APPS:                        "${IMAGE_PREFIX}apps:${VERSION}"
  JAVA:                        "${IMAGE_PREFIX}java:${VERSION}"
  DOCS:                        "${IMAGE_PREFIX}docs:${VERSION}"
  

# base image
@@ -49,7 +59,7 @@ check format:
################################################################################
gkfs:
  stage: build
  image: gekkofs/deps:0.9.6
  image: ${DEPS}
  interruptible: true
  needs: []
  script:
@@ -88,7 +98,7 @@ gkfs:
## == tests for scripts ====================
scripts:
  stage: test
  image: gekkofs/testing:0.9.6
  image: ${TESTING}
  needs: []
  script:
    - mkdir -p ${BUILD_PATH}/tests/scripts
@@ -100,10 +110,10 @@ scripts:
    reports:
      junit: ${BUILD_PATH}/tests/scripts/report.xml

## == integration tests for gkfs ===========
gkfs:allintegration:
## == integration tests for gkfs (group 1: I/O, FUSE, shell) ================
gkfs:integration-1:
  stage: test
  image: gekkofs/testing:0.9.6
  image: ${TESTING}
  interruptible: true
  needs: ['gkfs']

@@ -113,32 +123,30 @@ gkfs:allintegration:
    - mkdir -p ${BUILD_PATH}/tests/run
    - cd ${BUILD_PATH}/tests/integration
    - ${PYTEST} -v -n $(nproc)
          ${INTEGRATION_TESTS_BIN_PATH}/
          ${INTEGRATION_TESTS_BIN_PATH}/data
          ${INTEGRATION_TESTS_BIN_PATH}/fuse
          ${INTEGRATION_TESTS_BIN_PATH}/shell
          ${INTEGRATION_TESTS_BIN_PATH}/compatibility
          ${INTEGRATION_TESTS_BIN_PATH}/startup
          ${INTEGRATION_TESTS_BIN_PATH}/error_handling
          --basetemp=${BUILD_PATH}/tests/run/
          --junit-xml=report.xml
          --junit-xml=report-1.xml

    ## capture coverage information for this test and write it to
    ## $COVERAGE_PATH/$SUBTEST.info
    ## capture coverage information
    - cd ${CI_PROJECT_DIR}
    # use ccache
    - /usr/sbin/update-ccache-symlinks
    - export PATH="/usr/lib/ccache:$PATH"
    - cmake --preset ci-coverage
          -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}
          -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration.info
    ## Since the pipeline recreates the source tree, the access times for .gcno
    ## files are newer than those of .gcda files. This makes gcov emit a
    ## warning for each file which slows it down. Updating the timestamps
    ## avoids this
          -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration1.info
    - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \;
    - cmake --build ${BUILD_PATH} --target coverage-capture

  # fix relative paths so that GitLab can find the correct files
  after_script:
    - perl -i.orig
          -pe 's%file="(.*?)"%file="tests/integration/$1"%g;'
          -pe 's%(../)+install/share/gkfs/%%g;'
          ${BUILD_PATH}/tests/integration/report.xml
          ${BUILD_PATH}/tests/integration/report-1.xml

  artifacts:
    expire_in: 1 day
@@ -146,17 +154,67 @@ gkfs:allintegration:
    paths:
      - ${BUILD_PATH}
    reports:
      junit: ${BUILD_PATH}/tests/integration/report.xml
      junit: ${BUILD_PATH}/tests/integration/report-1.xml


gkfs:integration-2:
  stage: test
  image: ${TESTING}
  interruptible: true
  needs: ['gkfs']

  script:
    ## run tests
    - export PATH=${PATH}:/usr/local/bin
    - mkdir -p ${BUILD_PATH}/tests/run
    - cd ${BUILD_PATH}/tests/integration
    - ${PYTEST} -v -n $(nproc)
          ${INTEGRATION_TESTS_BIN_PATH}/syscalls
          ${INTEGRATION_TESTS_BIN_PATH}/directories
          ${INTEGRATION_TESTS_BIN_PATH}/operations
          ${INTEGRATION_TESTS_BIN_PATH}/rename
          ${INTEGRATION_TESTS_BIN_PATH}/position
          ${INTEGRATION_TESTS_BIN_PATH}/status
          ${INTEGRATION_TESTS_BIN_PATH}/malleability
          ${INTEGRATION_TESTS_BIN_PATH}/forwarding
          ${INTEGRATION_TESTS_BIN_PATH}/concurrency
          ${INTEGRATION_TESTS_BIN_PATH}/resilience
          --basetemp=${BUILD_PATH}/tests/run/
          --junit-xml=report-2.xml

    ## capture coverage information
    - cd ${CI_PROJECT_DIR}
    - /usr/sbin/update-ccache-symlinks
    - export PATH="/usr/lib/ccache:$PATH"
    - cmake --preset ci-coverage
          -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}
          -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration2.info
    - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \;
    - cmake --build ${BUILD_PATH} --target coverage-capture

  after_script:
    - perl -i.orig
          -pe 's%file="(.*?)"%file="tests/integration/$1"%g;'
          -pe 's%(../)+install/share/gkfs/%%g;'
          ${BUILD_PATH}/tests/integration/report-2.xml

  artifacts:
    expire_in: 1 day
    when: always
    paths:
      - ${BUILD_PATH}
    reports:
      junit: ${BUILD_PATH}/tests/integration/report-2.xml

## == integration tests for gkfs ===========
gkfs:integration:
  stage: test
  image: gekkofs/testing:0.9.6
  image: ${TESTING}
  interruptible: true
  needs: ['gkfs']  # we need to remove gkfs dependencies on manual
  parallel:
    matrix:
      - SUBTEST: [ data, status, syscalls, directories, operations, position, shell, rename, migration, startup, error_handling, resilience, compatibility ]
      - SUBTEST: [ data, status, syscalls, directories, operations, position, shell, rename, migration, startup, error_handling, resilience, compatibility, fuse ]
  rules:
    - if: '$CI_MERGE_REQUEST_EVENT_TYPE == "detached"'
      when: never
@@ -209,7 +267,7 @@ gkfs:integration:
## == integration tests for gkfwd ==========
gkfwd:integration:
  stage: test
  image: gekkofs/testing:0.9.6
  image: ${TESTING}
  interruptible: true
  needs: ['gkfs']
  parallel:
@@ -267,7 +325,8 @@ gkfwd:integration:
## == unit tests for gkfs ==================
gkfs:unit:
  stage: test
  image: gekkofs/testing:0.9.6
  image: ${TESTING}
  interruptible: true
  needs: ['gkfs']
  script:
    ## Add path to mkfs.kreon
@@ -307,7 +366,8 @@ gkfs:unit:
## == unit tests for gkfs ==================
gkfs:app:
  stage: test
  image: gekkofs/apps:0.9.6
  image: ${APPS}
  interruptible: true
  needs: ['gkfs']
  script:
    ## Add path to mkfs.kreon
@@ -348,7 +408,8 @@ gkfs:app:
## == java tests for gkfs ==================
gkfs:java:
  stage: test
  image: gekkofs/java:0.9.6
  image: ${JAVA}
  interruptible: true
  needs: ['gkfs']
  script:
    ## Add path to mkfs.kreon
@@ -389,7 +450,8 @@ gkfs:java:
## == python tests for gkfs ==================
gkfs:python:
  stage: test
  image: gekkofs/testing:0.9.6
  image: ${TESTING}
  interruptible: true
  needs: ['gkfs']
  script:
    ## Add path to mkfs.kreon
@@ -432,7 +494,7 @@ gkfs:python:
################################################################################
documentation:
  stage: docs
  image: gekkofs/docs:0.9.6
  image: ${DOCS}
  needs: []
  rules:
    # we only build the documentation automatically if we are on the
@@ -465,7 +527,7 @@ documentation:
## == coverage baseline ====================
coverage:baseline:
  stage: report
  image: gekkofs/testing:0.9.6
  image: ${TESTING}
  interruptible: true
  needs: ['gkfs']

@@ -496,9 +558,9 @@ coverage:baseline:

coverage:
  stage: report
  image: gekkofs/testing:0.9.6
  image: ${TESTING}
  #needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration']
  needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ]
  needs: [ 'coverage:baseline', 'gkfs:integration-1', 'gkfs:integration-2', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ]
  script:
      # use ccache
    - ccache --zero-stats
@@ -528,7 +590,7 @@ coverage:

cppcheck:
  stage: report
  image: gekkofs/testing:0.9.6
  image: ${TESTING}
  needs: [ 'gkfs']
  script:
    - cd ${CI_PROJECT_DIR}
+0 −3
Original line number Diff line number Diff line
[submodule "external/hermes"]
	path = external/hermes
	url = https://github.com/gekkofs/hermes.git
[submodule "tests/scripts/bats"]
	path = tests/scripts/bats
	url = https://github.com/bats-core/bats-core.git
+10 −0
Original line number Diff line number Diff line
@@ -28,14 +28,24 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
    - New sfind filtering in the server side
  - Added new tests (and enabling failing ones) to increase coverage
  - Added docker image (release) for 0.9.6 ([!279](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/279))
  - Added FUSE support([!264](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/264))
    - fuse_client mounting a virtual path exists, avoiding usage of LD_PRELOAD.
    - Added tests for FUSE support
    


### Changed 
  - Disabled at_parent/at_fork/at_child as it seems unneded now 
  - Moved some CMAKE options to config.hpp and env variables ([!285](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/285))
    - LIBGKFS/ GKFS _SYMLINK_SUPPORT, _RENAME_SUPPORT and _CREATE_CHECK_PARENTS.
    - Now all the performance options are in config.hpp and env variables.

### Fixed
  - SYS_lstat does not exists on some architectures, change to newfstatat ([!269](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/269))
    - We cannot use lstat directly as may cause a recursion call on libc interception.
  - Un/Packing order of directory entries in compressed format was incorrect ([!281](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/281))
  - Fix pytorch mmap ([!291](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/291))
    

## [0.9.5] - 2025-08
### New
+139 −0
Original line number Diff line number Diff line
# FindRocksDB
# ---------
#
# Find RocksDB include dirs and libraries.
#
# Use this module by invoking find_package with the form::
#
#   find_package(RocksDB
#     [version] [EXACT]     # Minimum or EXACT version e.g. 6.11.4
#     [REQUIRED]            # Fail with error if RocksDB is not found
#     )
#
# Imported Targets
# ^^^^^^^^^^^^^^^^
#
# This module provides the following imported targets, if found:
#
# ``RocksDB::rocksdb``
#   The RocksDB library
#
# Result Variables
# ^^^^^^^^^^^^^^^^
#
# This will define the following variables:
#
# ``RocksDB_FOUND``
#   True if the system has the RocksDB library.
# ``RocksDB_VERSION``
#   The version of the RocksDB library which was found.
# ``RocksDB_INCLUDE_DIRS``
#   Include directories needed to use RocksDB.
# ``RocksDB_LIBRARIES``
#   Libraries needed to link to RocksDB.
#
# Cache Variables
# ^^^^^^^^^^^^^^^
#
# The following cache variables may also be set:
#
# ``ROCKSDB_INCLUDE_DIR``
#   The directory containing ``rocksdb/db.h``.
# ``ROCKSDB_LIBRARY``
#   The path to the RocksDB library.
#

# Standard names to search for
set(RocksDB_NAMES rocksdb rocksdb_static)

if (NOT PKG_CONFIG_FOUND)
    find_package(PkgConfig)
endif ()

if (PKG_CONFIG_FOUND)
    pkg_check_modules(PC_ROCKSDB QUIET rocksdb)

    find_path(ROCKSDB_INCLUDE_DIR
              NAMES rocksdb/db.h
              HINTS ${PC_ROCKSDB_INCLUDE_DIRS}
              PATH_SUFFIXES include)

    find_library(ROCKSDB_LIBRARY_RELEASE
                 NAMES ${RocksDB_NAMES}
                 HINTS ${PC_ROCKSDB_LIBRARY_DIRS}
                 PATH_SUFFIXES lib)

    set(RocksDB_VERSION_STRING ${PC_ROCKSDB_VERSION})
else()
    find_path(ROCKSDB_INCLUDE_DIR
              NAMES rocksdb/db.h
              PATH_SUFFIXES include)

    find_library(ROCKSDB_LIBRARY_RELEASE
                 NAMES ${RocksDB_NAMES}
                 PATH_SUFFIXES lib)
endif()

# Allow ROCKSDB_LIBRARY to be set manually, as the location of the rocksdb library
if(NOT ROCKSDB_LIBRARY)
  include(SelectLibraryConfigurations)
  select_library_configurations(ROCKSDB)
endif()

if(NOT ROCKSDB_LIBRARY)
  set(ROCKSDB_LIBRARY ${ROCKSDB_LIBRARY_RELEASE})
endif()

mark_as_advanced(ROCKSDB_INCLUDE_DIR ROCKSDB_LIBRARY)

include(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(RocksDB
        REQUIRED_VARS ROCKSDB_LIBRARY ROCKSDB_INCLUDE_DIR
        VERSION_VAR RocksDB_VERSION_STRING)

if(RocksDB_FOUND)
    set(RocksDB_INCLUDE_DIRS ${ROCKSDB_INCLUDE_DIR})

    if(NOT RocksDB_LIBRARIES)
        set(RocksDB_LIBRARIES ${ROCKSDB_LIBRARY})
    endif()

    if(NOT TARGET RocksDB::rocksdb)
        add_library(RocksDB::rocksdb UNKNOWN IMPORTED)
        set_target_properties(RocksDB::rocksdb PROPERTIES
                INTERFACE_INCLUDE_DIRECTORIES "${RocksDB_INCLUDE_DIRS}")
       
        set_target_properties(RocksDB::rocksdb PROPERTIES
                IMPORTED_LOCATION "${ROCKSDB_LIBRARY}")

        find_library(LZ4_LIBRARY NAMES lz4)
        if(LZ4_LIBRARY)
            target_link_libraries(RocksDB::rocksdb INTERFACE ${LZ4_LIBRARY})
        endif()

        find_library(SNAPPY_LIBRARY NAMES snappy)
        if(SNAPPY_LIBRARY)
            target_link_libraries(RocksDB::rocksdb INTERFACE ${SNAPPY_LIBRARY})
        endif()

        find_library(ZSTD_LIBRARY NAMES zstd)
        if(ZSTD_LIBRARY)
            target_link_libraries(RocksDB::rocksdb INTERFACE ${ZSTD_LIBRARY})
        endif()

        find_library(ZLIB_LIBRARY NAMES z)
        if(ZLIB_LIBRARY)
            target_link_libraries(RocksDB::rocksdb INTERFACE ${ZLIB_LIBRARY})
        endif()

        find_library(BZIP2_LIBRARY NAMES bz2)
        if(BZIP2_LIBRARY)
            target_link_libraries(RocksDB::rocksdb INTERFACE ${BZIP2_LIBRARY})
        endif()

        find_library(URING_LIBRARY NAMES uring)
        if(URING_LIBRARY)
            target_link_libraries(RocksDB::rocksdb INTERFACE ${URING_LIBRARY})
        endif()
    endif()
endif()
+167 −0
Original line number Diff line number Diff line
# FindThallium
# ---------
#
# Find Thallium include dirs and libraries.
#
# Use this module by invoking find_package with the form::
#
#   find_package(Thallium
#     [version] [EXACT]     # Minimum or EXACT version e.g. 0.6.2
#     [REQUIRED]            # Fail with error if Thallium is not found
#     )
#
# Imported Targets
# ^^^^^^^^^^^^^^^^
#
# This module provides the following imported targets, if found:
#
# ``Thallium::Thallium``
#   The Thallium library
#
# Result Variables
# ^^^^^^^^^^^^^^^^
#
# This will define the following variables:
#
# ``Thallium_FOUND``
#   True if the system has the Thallium library.
# ``Thallium_VERSION``
#   The version of the Thallium library which was found.
# ``Thallium_INCLUDE_DIRS``
#   Include directories needed to use Thallium.
#
# Cache Variables
# ^^^^^^^^^^^^^^^
#
# The following cache variables may also be set:
#
# ``THALLIUM_INCLUDE_DIR``
#   The directory containing ``thallium.hpp``.
#

function(_get_pkgconfig_paths target_var)
    set(_lib_dirs)
    if (NOT DEFINED CMAKE_SYSTEM_NAME
        OR (CMAKE_SYSTEM_NAME MATCHES "^(Linux|kFreeBSD|GNU)$"
        AND NOT CMAKE_CROSSCOMPILING)
    )
        if (EXISTS "/etc/debian_version") # is this a debian system ?
            if (CMAKE_LIBRARY_ARCHITECTURE)
                list(APPEND _lib_dirs "lib/${CMAKE_LIBRARY_ARCHITECTURE}/pkgconfig")
            endif ()
        else ()
            # not debian, check the FIND_LIBRARY_USE_LIB32_PATHS and FIND_LIBRARY_USE_LIB64_PATHS properties
            get_property(uselib32 GLOBAL PROPERTY FIND_LIBRARY_USE_LIB32_PATHS)
            if (uselib32 AND CMAKE_SIZEOF_VOID_P EQUAL 4)
                list(APPEND _lib_dirs "lib32/pkgconfig")
            endif ()
            get_property(uselib64 GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS)
            if (uselib64 AND CMAKE_SIZEOF_VOID_P EQUAL 8)
                list(APPEND _lib_dirs "lib64/pkgconfig")
            endif ()
            get_property(uselibx32 GLOBAL PROPERTY FIND_LIBRARY_USE_LIBX32_PATHS)
            if (uselibx32 AND CMAKE_INTERNAL_PLATFORM_ABI STREQUAL "ELF X32")
                list(APPEND _lib_dirs "libx32/pkgconfig")
            endif ()
        endif ()
    endif ()
    if (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD" AND NOT CMAKE_CROSSCOMPILING)
        list(APPEND _lib_dirs "libdata/pkgconfig")
    endif ()
    list(APPEND _lib_dirs "lib/pkgconfig")
    list(APPEND _lib_dirs "share/pkgconfig")

    set(_extra_paths)
    list(APPEND _extra_paths ${CMAKE_PREFIX_PATH})
    list(APPEND _extra_paths ${CMAKE_FRAMEWORK_PATH})
    list(APPEND _extra_paths ${CMAKE_APPBUNDLE_PATH})

    # Check if directories exist and eventually append them to the
    # pkgconfig path list
    foreach (_prefix_dir ${_extra_paths})
        foreach (_lib_dir ${_lib_dirs})
            if (EXISTS "${_prefix_dir}/${_lib_dir}")
                list(APPEND _pkgconfig_paths "${_prefix_dir}/${_lib_dir}")
                list(REMOVE_DUPLICATES _pkgconfig_paths)
            endif ()
        endforeach ()
    endforeach ()

    set("${target_var}"
        ${_pkgconfig_paths}
        PARENT_SCOPE
    )
endfunction()

if (NOT PKG_CONFIG_FOUND)
    find_package(PkgConfig)
endif ()

if (PKG_CONFIG_FOUND)
    pkg_check_modules(PC_THALLIUM QUIET thallium)

    find_path(
        THALLIUM_INCLUDE_DIR
        NAMES thallium.hpp
        PATHS ${PC_THALLIUM_INCLUDE_DIRS}
        PATH_SUFFIXES include
    )

    set(Thallium_VERSION ${PC_THALLIUM_VERSION})
else ()
    find_path(
        THALLIUM_INCLUDE_DIR
        NAMES thallium.hpp
        PATH_SUFFIXES include
    )

    # even if pkg-config is not available, try to find thallium.pc
    _get_pkgconfig_paths(_pkgconfig_paths)

    find_file(_thallium_pc_file thallium.pc PATHS "${_pkgconfig_paths}")

    if (NOT _thallium_pc_file)
        message(
            FATAL_ERROR
            "ERROR: Could not find 'thallium.pc' file. Unable to determine library version"
        )
    endif ()

    file(STRINGS "${_thallium_pc_file}" _thallium_pc_file_contents REGEX "Version: ")

    if ("${_thallium_pc_file_contents}" MATCHES "Version: ([0-9]+\\.[0-9]+\\.[0-9]+)")
        set(Thallium_VERSION ${CMAKE_MATCH_1})
    else ()
        message(FATAL_ERROR "ERROR: Failed to determine library version")
    endif ()

    unset(_pkgconfig_paths)
    unset(_thallium_pc_file_contents)
endif ()

mark_as_advanced(THALLIUM_INCLUDE_DIR)

include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(
    Thallium
    FOUND_VAR Thallium_FOUND
    REQUIRED_VARS THALLIUM_INCLUDE_DIR
    VERSION_VAR Thallium_VERSION
)

if (Thallium_FOUND)
    set(Thallium_INCLUDE_DIRS ${THALLIUM_INCLUDE_DIR})
    if (NOT TARGET thallium)
        add_library(thallium INTERFACE IMPORTED)
        set_target_properties(
            thallium
            PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${THALLIUM_INCLUDE_DIR}"
        )
        
        # Thallium wraps Margo, Mercury, and Argobots
        find_package(Margo REQUIRED)
        find_package(Mercury REQUIRED)
        find_package(Argobots REQUIRED)
        target_link_libraries(thallium INTERFACE Margo::Margo Mercury::Mercury Argobots::Argobots)
    endif ()
endif ()
Loading