Commit 39767a16 authored by Ramon Nou's avatar Ramon Nou
Browse files

Malleability extension -> mutate (shrink, expand) and random slicing + cutshift capabilities

parent 247b6ec8

.gitlab-ci.yml

+51 −5
Original line number Diff line number Diff line
@@ -181,10 +181,6 @@ gkfs:integration-2: 
          ${INTEGRATION_TESTS_BIN_PATH}/rename

          ${INTEGRATION_TESTS_BIN_PATH}/position

          ${INTEGRATION_TESTS_BIN_PATH}/status

          ${INTEGRATION_TESTS_BIN_PATH}/malleability

          ${INTEGRATION_TESTS_BIN_PATH}/forwarding

          ${INTEGRATION_TESTS_BIN_PATH}/concurrency

          ${INTEGRATION_TESTS_BIN_PATH}/resilience

          --basetemp=${BUILD_PATH}/tests/run/

          --junit-xml=report-2.xml
@@ -212,6 +208,56 @@ gkfs:integration-2: 
    reports:

      junit: ${BUILD_PATH}/tests/integration/report-2.xml





gkfs:integration-3:

  stage: test

  image: ${TESTING}

  interruptible: true

  needs: ['gkfs']



  script:

    ## run tests

    - export PATH=${PATH}:/usr/local/bin

    - export GKFS_MALLEABILITY_CI_FAST=ON

    - export GKFS_PERF_REPETITIONS=1

    - export GKFS_PERF_NUM_FILES=1

    - export GKFS_PERF_FILE_SIZE=$((1024 * 1024))

    - export GKFS_MALLEABILITY_OLD_NODES=2

    - export GKFS_MALLEABILITY_NEW_NODES=1

    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - ${PYTEST} -v -n $(nproc)

          ${INTEGRATION_TESTS_BIN_PATH}/malleability

          ${INTEGRATION_TESTS_BIN_PATH}/forwarding

          ${INTEGRATION_TESTS_BIN_PATH}/concurrency

          ${INTEGRATION_TESTS_BIN_PATH}/resilience

          --basetemp=${BUILD_PATH}/tests/run/

          --junit-xml=report-3.xml



    ## capture coverage information

    - cd ${CI_PROJECT_DIR}

    - /usr/sbin/update-ccache-symlinks

    - export PATH="/usr/lib/ccache:$PATH"

    - cmake --preset ci-coverage

          -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}

          -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration3.info

    - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \;

    - cmake --build ${BUILD_PATH} --target coverage-capture



  after_script:

    - perl -i.orig

          -pe 's%file="(.*?)"%file="tests/integration/$1"%g;'

          -pe 's%(../)+install/share/gkfs/%%g;'

          ${BUILD_PATH}/tests/integration/report-3.xml



  artifacts:

    expire_in: 1 day

    when: always

    paths:

      - ${BUILD_PATH}

    reports:

      junit: ${BUILD_PATH}/tests/integration/report-3.xml



## == integration tests for gkfs ===========

gkfs:integration:

  stage: test
@@ -566,7 +612,7 @@ coverage: 
  stage: report

  image: ${TESTING}

  #needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration']

  needs: [ 'coverage:baseline', 'gkfs:integration-1', 'gkfs:integration-2', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ]

  needs: [ 'coverage:baseline', 'gkfs:integration-1', 'gkfs:integration-2', 'gkfs:integration-3', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ]

  script:

      # use ccache

    - ccache --zero-stats

CHANGELOG.md

+8 −2
Original line number Diff line number Diff line
@@ -47,8 +47,11 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 
  - Client interface and hostfile removel ([!313](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/313))

    - Demon option to avoid removing hostfile on exit : ENV variable , GKFS_KEEP_HOSTS_FILE or option --keep-hosts

    - Client Interface selection avoiding loopback setup : LIBGKFS_OFI_INTERFACE=ib0

     

 

  - Random Slicing + cutshift ([!316](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/316))

    - Added marker-based expand, shrink, and mixed mutate workflow using one shared hostfile.

    - Added Random Slicing placement with CutShift for expand-only resizes to reduce inter-node data movement.

    - Added optional expand-on-demand data movement for pure expand operations.

    - Added malleability integration coverage and user documentation.





### Changed
@@ -60,6 +63,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 
    - LIBGKFS/ GKFS _SYMLINK_SUPPORT, _RENAME_SUPPORT and _CREATE_CHECK_PARENTS.

    - Now all the performance options are in config.hpp and env variables.

  - Added DIRECT_IO on server side to increase performance ([!312](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/312))

  - Shared metrics/message routing variables now use the common `GKFS_` prefix.





### Fixed

  - SYS_lstat does not exists on some architectures, change to newfstatat ([!269](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/269))
@@ -71,6 +76,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 
  - Fix remove chunk bug ([!294](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/294))

  - Fix decompress_and_parse_entries_standard() Unexpected end of buffer while parsing name bug ([!312](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/312))

  - Fix client dissapearing on malleability ends with an error. ([!313](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/313))

  - Fixed buffer-size validation issues in filtered directory listing paths.

    



## [0.9.5] - 2025-08

README.md

+335 −102

File changed.

Preview size limit exceeded, changes collapsed.

examples/gfind/sfind.cpp

+12 −4
Original line number Diff line number Diff line
@@ -302,11 +302,15 @@ worker_routine(void* arg) { 


                if(n > 0 && entries && !data->opt->just_count) {

                    char* ptr = reinterpret_cast<char*>(entries);

                    int bytes_processed = 0;

                    size_t bytes_processed = 0;

                    while(bytes_processed < n) {

                        struct dirent_extended* temp =

                                reinterpret_cast<struct dirent_extended*>(ptr);

                        if(temp->d_reclen == 0)

                        const auto remaining =

                                static_cast<size_t>(n) - bytes_processed;

                        if(temp->d_reclen <

                                   offsetof(struct dirent_extended, d_name) + 1 ||

                           temp->d_reclen > remaining)

                            break;



                        local_found++;
@@ -360,11 +364,15 @@ worker_routine(void* arg) { 
            }



            char* ptr = reinterpret_cast<char*>(entries);

            int bytes_processed = 0;

            size_t bytes_processed = 0;

            while(bytes_processed < n) {

                struct dirent_extended* temp =

                        reinterpret_cast<struct dirent_extended*>(ptr);

                if(temp->d_reclen == 0)

                const auto remaining =

                        static_cast<size_t>(n) - bytes_processed;

                if(temp->d_reclen <

                           offsetof(struct dirent_extended, d_name) + 1 ||

                   temp->d_reclen > remaining)

                    break;



                if(temp->d_type != 1) {

include/client/env.hpp

+5 −6
Original line number Diff line number Diff line
@@ -83,8 +83,6 @@ static constexpr auto PROTECT_FILES_CONSUMER = 
        ADD_PREFIX("PROTECT_FILES_CONSUMER");

static constexpr auto RANGE_FD = ADD_PREFIX("RANGE_FD");

static constexpr auto DIRENTS_BUFF_SIZE = ADD_PREFIX("DIRENTS_BUFF_SIZE");

static constexpr auto USE_DIRENTS_COMPRESSION =

        ADD_PREFIX("USE_DIRENTS_COMPRESSION");



static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL");

static constexpr auto PROXY_PID_FILE = ADD_PREFIX("PROXY_PID_FILE");
@@ -95,10 +93,6 @@ static constexpr auto WRITE_SIZE_THRESHOLD = 
        ADD_PREFIX("WRITE_SIZE_CACHE_THRESHOLD");

} // namespace cache



static constexpr auto CREATE_CHECK_PARENTS = ADD_PREFIX("CREATE_CHECK_PARENTS");

static constexpr auto SYMLINK_SUPPORT = ADD_PREFIX("SYMLINK_SUPPORT");

static constexpr auto RENAME_SUPPORT = ADD_PREFIX("RENAME_SUPPORT");

static constexpr auto USE_INLINE_DATA = ADD_PREFIX("USE_INLINE_DATA");

static constexpr auto CREATE_WRITE_OPTIMIZATION =

        ADD_PREFIX("CREATE_WRITE_OPTIMIZATION");

static constexpr auto READ_INLINE_PREFETCH = ADD_PREFIX("READ_INLINE_PREFETCH");
@@ -107,6 +101,11 @@ static constexpr auto METADATA_BATCH = ADD_PREFIX("METADATA_BATCH"); 
static constexpr auto METADATA_BATCH_THRESHOLD =

        ADD_PREFIX("METADATA_BATCH_THRESHOLD");

static constexpr auto ASYNC_WRITE = ADD_PREFIX("ASYNC_WRITE");

static constexpr auto FUSE_ENTRY_TIMEOUT = ADD_PREFIX("FUSE_ENTRY_TIMEOUT");

static constexpr auto FUSE_ATTR_TIMEOUT = ADD_PREFIX("FUSE_ATTR_TIMEOUT");

static constexpr auto FUSE_NEGATIVE_TIMEOUT =

        ADD_PREFIX("FUSE_NEGATIVE_TIMEOUT");

static constexpr auto FUSE_WRITEBACK = ADD_PREFIX("FUSE_WRITEBACK");



// Libfabric interface pinning (consumed by libfabric at HG_init() time)

// OFI_INTERFACE is used with the GKFS_ prefix (e.g., LIBGKFS_OFI_INTERFACE)