Commit 2e0d6906 authored by Ramon Nou's avatar Ramon Nou
Browse files

fixes and refactor: preparing for 0.9.6 release

parent 59724f6e

.clang-tidy

0 → 100644
+4 −0
Changes for .clang-tidy: 4 added lines, 0 removed lines.
Original line number Diff line number Diff line 
Checks: '-*,clang-analyzer-*'

WarningsAsErrors: ''

HeaderFilterRegex: '(^|/)(src|include)/'

FormatStyle: none
 
 No newline at end of file

.gitignore

+2 −0
Changes for .gitignore: 2 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -98,3 +98,5 @@ CMakeUserPresets.json 
gkfs/

.gitlab-ci-local/

install-*-test/

perf-results/

/WIP.md

.gitlab-ci.yml

+94 −28
Changes for .gitlab-ci.yml: 94 added lines, 28 removed lines.
Original line number Diff line number Diff line
@@ -15,7 +15,6 @@ variables: 
  INTEGRATION_TESTS_BIN_PATH:   "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration"

  COVERAGE_PATH:                "${CI_PROJECT_DIR}/gkfs/build/coverage"

  PYTEST:                       "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test"

  BATS:                         "${CI_PROJECT_DIR}/tests/scripts/bats/bin/bats"

  LD_LIBRARY_PATH:              "${CI_PROJECT_DIR}/deps/install/lib:${CI_PROJECT_DIR}/deps/install/lib64"

  # Configuration variables

  GKFS_LOG_LEVEL:               "100"
@@ -54,6 +53,14 @@ check format: 
      -v





check rpc consistency:

  stage: lint

  image: ${DEPS}

  needs: []

  script:

    - python3 ${SCRIPTS_DIR}/check_rpc_consistency.py "${CI_PROJECT_DIR}"





################################################################################

## Building

################################################################################
@@ -95,27 +102,6 @@ gkfs: 
## Testing

################################################################################



## == tests for scripts ====================

scripts:

  stage: test

  image: ${TESTING}

  needs: []

  script:

    - mkdir -p ${BUILD_PATH}/tests/scripts

    - cd ${BUILD_PATH}/tests/scripts

    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/dl_dep.sh --formatter junit

    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/compile_dep.sh --formatter junit

  rules:

    - if: '$CI_MERGE_REQUEST_EVENT_TYPE == "detached"'

      when: never

    - if: '$CI_MERGE_REQUEST_ID != ""'

      when: manual

      allow_failure: true

  artifacts:

    expire_in: 1 day

    reports:

      junit: ${BUILD_PATH}/tests/scripts/report.xml



## == integration tests for gkfs (group 1: I/O, FUSE, shell) ================

gkfs:integration-1:

  stage: test
@@ -128,7 +114,7 @@ gkfs:integration-1: 
    - export PATH=${PATH}:/usr/local/bin

    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - ${PYTEST} -v -n $(nproc)

    - ${PYTEST} -v -n $(nproc) --dist=loadscope --timeout=900 --timeout-method=thread

          ${INTEGRATION_TESTS_BIN_PATH}/data

          ${INTEGRATION_TESTS_BIN_PATH}/fuse

          ${INTEGRATION_TESTS_BIN_PATH}/shell
@@ -174,7 +160,7 @@ gkfs:integration-2: 
    - export PATH=${PATH}:/usr/local/bin

    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - ${PYTEST} -v -n $(nproc)

    - ${PYTEST} -v -n $(nproc) --dist=loadscope --timeout=900 --timeout-method=thread

          ${INTEGRATION_TESTS_BIN_PATH}/syscalls

          ${INTEGRATION_TESTS_BIN_PATH}/directories

          ${INTEGRATION_TESTS_BIN_PATH}/operations
@@ -226,7 +212,7 @@ gkfs:integration-3: 
    - export GKFS_MALLEABILITY_NEW_NODES=1

    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - ${PYTEST} -v -n $(nproc)

    - ${PYTEST} -v -n $(nproc) --dist=loadscope --timeout=900 --timeout-method=thread

          ${INTEGRATION_TESTS_BIN_PATH}/malleability

          ${INTEGRATION_TESTS_BIN_PATH}/forwarding

          ${INTEGRATION_TESTS_BIN_PATH}/concurrency
@@ -278,7 +264,7 @@ gkfs:integration: 
    - export PATH=${PATH}:/usr/local/bin

    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - ${PYTEST} -v -n $(nproc)

    - ${PYTEST} -v -n $(nproc) --dist=loadscope --timeout=900 --timeout-method=thread

          ${INTEGRATION_TESTS_BIN_PATH}/${SUBTEST}

          --basetemp=${BUILD_PATH}/tests/run/${SUBTEST}

          --junit-xml=report.xml
@@ -336,7 +322,7 @@ gkfwd:integration: 
    ## run tests

    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - ${PYTEST} -v -n $(nproc)

    - ${PYTEST} -v -n $(nproc) --dist=loadscope --timeout=900 --timeout-method=thread

          ${INTEGRATION_TESTS_BIN_PATH}/${SUBTEST}

          --basetemp=${BUILD_PATH}/tests/run/${SUBTEST}

          --junit-xml=report.xml
@@ -456,6 +442,25 @@ gkfs:app: 
    reports:

      junit: ${BUILD_PATH}/tests/apps/report.xml



compatibility-corpus:

  stage: test

  image: ${TESTING}

  needs: ['gkfs']

  script:

    - cd ${CI_PROJECT_DIR}

    - python3 scripts/compatibility_corpus.py

          --daemon ${INSTALL_PATH}/bin/gkfs_daemon

          --client ${BUILD_PATH}/tests/integration/harness/gkfs.io

          --library ${INSTALL_PATH}/lib/libgkfs_intercept.so

          --libc-library ${INSTALL_PATH}/lib/libgkfs_libc_intercept.so

          --user-library ${INSTALL_PATH}/lib/libgkfs_user_lib.so

          --output ${BUILD_PATH}/tests/compatibility/corpus.json

  artifacts:

    when: always

    paths:

      - ${BUILD_PATH}/tests/compatibility/corpus.json

    expire_in: 5 days





## == java tests for gkfs ==================

gkfs:java:
@@ -652,12 +657,74 @@ cppcheck: 
  after_script:

    - perl -pe 's|/builds/gitlab/hpc/gekkofs||g' cppcheck.json > cppcheck2.json

  artifacts:

    expose_as: "Cppcheck report"

    paths:

      - cppcheck_out.xml

      - cppcheck2.json

    reports:

      codequality: cppcheck2.json

    expire_in: 5 days



static-analysis:

  stage: report

  image: ${TESTING}

  needs: ['gkfs']

  script:

    - cd ${CI_PROJECT_DIR}

    - ${SCRIPTS_DIR}/check_static_analysis.sh --cppcheck --require-tools

          --build-dir ${BUILD_PATH} --output-dir analysis

    - ${SCRIPTS_DIR}/check_static_analysis.sh --clang-tidy --require-tools

          --build-dir ${BUILD_PATH} --output-dir analysis

          --clang-tidy-files

          src/common/rpc/distributor.cpp

          src/common/rpc/distributor_factory.cpp

          src/common/rpc/distribution_config.cpp

          src/daemon/classes/fs_data.cpp

          src/daemon/handler/srv_malleability.cpp

          src/client/rpc/forward_malleability.cpp

  artifacts:

    when: always

    paths:

      - analysis

    expire_in: 5 days



reports:index:

  stage: report

  image: ${TESTING}

  needs:

    - job: coverage

      artifacts: true

    - job: cppcheck

      artifacts: true

    - job: static-analysis

      artifacts: true

  when: always

  script:

    - cd ${CI_PROJECT_DIR}

    - python3 ${SCRIPTS_DIR}/ci/generate_report_index.py

  artifacts:

    expose_as: "CI reports"

    when: always

    paths:

      - reports/

    expire_in: 5 days



sanitizers:

  stage: report

  image: ${DEPS}

  needs: []

  parallel:

    matrix:

      - SANITIZER: [asan, ubsan, tsan]

  script:

    - cd ${CI_PROJECT_DIR}

    - cmake --preset ci-${SANITIZER}

    - cmake --build ${CI_PROJECT_DIR}/gkfs/build-${SANITIZER}

          --target unit_tests --parallel $(nproc)

    - ctest --test-dir ${CI_PROJECT_DIR}/gkfs/build-${SANITIZER}

          --output-on-failure -L unit::all

          -E "number of chunks involved in an operation"





################################################################################

## Deployment of documentation and reports
@@ -681,4 +748,3 @@ deploy: 
        --chown=${DEPLOY_USERNAME}:${DEPLOY_GROUP}

        ${BUILD_PATH}/docs/sphinx/sphinx_docs/

        ${DEPLOY_USERNAME}@${DEPLOY_SERVER}:${DEPLOY_PATH}

.gitmodules

+0 −12
Changes for .gitmodules: 0 added lines, 12 removed lines.
Original line number Diff line number Diff line 
[submodule "tests/scripts/bats"]

	path = tests/scripts/bats

	url = https://github.com/bats-core/bats-core.git

[submodule "tests/scripts/helpers/bats-assert"]

	path = tests/scripts/helpers/bats-assert

	url = https://github.com/bats-core/bats-assert.git

[submodule "tests/scripts/helpers/bats-support"]

	path = tests/scripts/helpers/bats-support

	url = https://github.com/bats-core/bats-support.git

[submodule "tests/scripts/helpers/bats-file"]

	path = tests/scripts/helpers/bats-file

	url = https://github.com/bats-core/bats-file.git

[submodule "external/spdlog"]

	path = external/spdlog

	url = https://github.com/gabime/spdlog

CHANGELOG.md

+27 −0
Changes for CHANGELOG.md: 27 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -52,6 +52,18 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 
    - Added Random Slicing placement with CutShift for expand-only resizes to reduce inter-node data movement.

    - Added optional expand-on-demand data movement for pure expand operations.

    - Added malleability integration coverage and user documentation.

  - Preparation for 0.9.6 release ([!317](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/317))

    - Added runtime, operation, I/O, backend, migration, and latency metrics for Prometheus.

    - Added `GKFS_DAEMON_PROMETHEUS_AUTH=user:pass` for optional Pushgateway basic authentication.

    - Added unit coverage for metrics output and invalid authentication values.

  - Distributed RPC tracing and client RPC deadlines.

    - Added sampled trace correlation IDs across client, proxy, and daemon RPCs with `GKFS_ENABLE_TRACE` and `GKFS_TRACE_SAMPLE_RATE`.

    - Added optional client RPC deadlines through `LIBGKFS_RPC_TIMEOUT_MS` without retrying potentially non-idempotent operations.

  - Hostfile and malleability lifecycle metadata.

    - Added atomic hostfile updates, marker preservation, Random Slicing layout records, and layout hash validation across restart and mutate operations.

    - Added RAII file-descriptor handling and shared hostfile-management utilities.

  - Expanded diagnostics and engineering tooling.

    - Added RPC consistency checking, static-analysis helpers, compatibility-corpus workflows, logging-scale measurements, CI report indexing, and metadata reporting tools.





### Changed
@@ -64,6 +76,15 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 
    - Now all the performance options are in config.hpp and env variables.

  - Added DIRECT_IO on server side to increase performance ([!312](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/312))

  - Shared metrics/message routing variables now use the common `GKFS_` prefix.

  - Preparation for 0.9.6 release ([!317](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/317))

    - Reduced client logging prefixes and routed client startup summaries to the configured client log destination.

    - Standardized client and daemon log levels to full uppercase names such as `INFO`, `DEBUG`, and `ERROR`.

    - Periodic daemon statistics can be written to the configured daemon log file with `GKFS_DAEMON_STATS_LOG=ON`.

    - Updated external dependencies, including fmt, spdlog, CLI11, GSL, and MessagePack, and raised the minimum CMake version for current test tooling.

    - Added CMake presets and configuration reports for debug, release, coverage, and sanitizer builds.

    - Reworked integration and CI execution with deterministic workflows, pytest timeouts/load distribution, sanitizer coverage, and RPC consistency checks.

    - Removed the legacy Bats-based dependency-script test setup in favor of the Python testing workflows.

    - Improved client operation diagnostics, readiness handling, retry/disconnect behavior, descriptor management, and per-process/sample-based logging controls.





### Fixed
@@ -76,7 +97,13 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 
  - Fix remove chunk bug ([!294](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/294))

  - Fix decompress_and_parse_entries_standard() Unexpected end of buffer while parsing name bug ([!312](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/312))

  - Fix client dissapearing on malleability ends with an error. ([!313](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/313))

  - Preparation for 0.9.6 release ([!317](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/317))

    - Fixed buffer-size validation issues in filtered directory listing paths.

    - Prometheus operation counters now update when Prometheus is enabled, including `IOPS_WRITE` and `IOPS_READ`.

    - Reduced noisy daemon errors for expected missing sparse chunk files; `ENOENT` reads are logged at debug level.

    - Hardened RPC and proxy error propagation, startup readiness, daemon disconnect handling, and malleability/migration cleanup.

    - Fixed client syscall and symlink edge cases, descriptor lifecycle issues, and compatibility-corpus snapshot/report generation.

    - Hardened `sfind` directory-record parsing and filtered-directory handling.

      



## [0.9.5] - 2025-08