Commit 8675113b authored by Ramon Nou's avatar Ramon Nou
Browse files

Merge branch 'rnou/libc' into 'master' 

LIBC interception

Support for libc interception

See merge request !202
parents 2974fd23 e1c1aa7e

.gitlab-ci.yml

+76 −49
Original line number Diff line number Diff line
@@ -22,20 +22,20 @@ variables: 
  LIBGKFS_LOG:                  "all"

  LIBGKFS_LOG_OUTPUT:           "${CI_PROJECT_DIR}/logs/gkfs_client.log"

  GIT_SUBMODULE_STRATEGY:       recursive

  CCACHE_DIR:                   "$CI_PROJECT_DIR/ccache"

  CCACHE_DIR:                   "${CI_PROJECT_DIR}/ccache"

#  SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar"  # Defines the location of the analysis task cache

#  GIT_DEPTH: "0"  # Tells git to fetch all the branches of the project, required by the analysis task





# base image

image: gekkofs/core:0.9.4

image: gekkofs/core:0.9.5



################################################################################

## Validating

################################################################################

check format:

  stage: lint

  image: gekkofs/linter:0.9.4

  image: gekkofs/linter:0.9.5

  needs: []

  script:

    - ${SCRIPTS_DIR}/check_format.sh
@@ -49,18 +49,21 @@ check format: 
################################################################################

gkfs:

  stage: build

  image: gekkofs/deps:0.9.4

  image: gekkofs/deps:0.9.5

  interruptible: true

  needs: []

  script:

    # Change config.hpp with sed to enable extra features

    - sed -i 's/constexpr auto use_atime = false;/constexpr auto use_atime = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"

    #- sed -i 's/constexpr auto use_atime = false;/constexpr auto use_atime = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"

    - sed -i 's/constexpr auto use_ctime = false;/constexpr auto use_ctime = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"

    - sed -i 's/constexpr auto use_mtime = false;/constexpr auto use_mtime = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"

    - sed -i 's/constexpr auto use_link_cnt = false;/constexpr auto use_link_cnt = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"

    - sed -i 's/constexpr auto use_blocks = false;/constexpr auto use_blocks = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"

    - sed -i 's/constexpr bool use_dentry_cache = false;/constexpr bool use_dentry_cache = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"

    #- sed -i 's/constexpr auto zero_buffer_before_read = false;/constexpr auto zero_buffer_before_read = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"

    #- sed -i 's/constexpr auto implicit_data_removal = true;/constexpr auto implicit_data_removal = false;/g' "${CI_PROJECT_DIR}/include/config.hpp"

    # use ccache

    - ccache --zero-stats

    - ccache --zero-stats -M 750MiB -F 800 --evict-older-than 10d

    - /usr/sbin/update-ccache-symlinks

    - export PATH="/usr/lib/ccache:$PATH"

    - cmake --preset ci-coverage -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}
@@ -72,7 +75,7 @@ gkfs: 
    paths:

      - ${BUILD_PATH}

      - ${INSTALL_PATH}

    expire_in: 1 week

    expire_in: 1 day

  cache:

    key: ccache-gekko

    paths:
@@ -85,7 +88,7 @@ gkfs: 
## == tests for scripts ====================

scripts:

  stage: test

  image: gekkofs/testing:0.9.4

  image: gekkofs/testing:0.9.5

  needs: []

  script:

    - mkdir -p ${BUILD_PATH}/tests/scripts
@@ -93,14 +96,14 @@ scripts: 
    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/dl_dep.sh --formatter junit

    - ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/compile_dep.sh --formatter junit

  artifacts:

    expire_in: 1 week

    expire_in: 1 day

    reports:

      junit: ${BUILD_PATH}/tests/scripts/report.xml



## == integration tests for gkfs ===========

gkfs:allintegration:

  stage: test

  image: gekkofs/testing:0.9.4

  image: gekkofs/testing:0.9.5

  interruptible: true

  needs: ['gkfs']
@@ -138,7 +141,7 @@ gkfs:allintegration: 
          ${BUILD_PATH}/tests/integration/report.xml



  artifacts:

    expire_in: 1 week

    expire_in: 1 day

    when: always

    paths:

      - ${BUILD_PATH}
@@ -148,7 +151,7 @@ gkfs:allintegration: 
## == integration tests for gkfs ===========

gkfs:integration:

  stage: test

  image: gekkofs/testing:0.9.4

  image: gekkofs/testing:0.9.5

  interruptible: true

  needs: ['gkfs']  # we need to remove gkfs dependencies on manual

  parallel:
@@ -179,6 +182,7 @@ gkfs:integration: 
    - cmake --preset ci-coverage

          -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}

          -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/${SUBTEST}.info

          -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external

    ## Since the pipeline recreates the source tree, the access times for .gcno

    ## files are newer than those of .gcda files. This makes gcov emit a

    ## warning for each file which slows it down. Updating the timestamps
@@ -194,7 +198,7 @@ gkfs:integration: 
          ${BUILD_PATH}/tests/integration/report.xml



  artifacts:

    expire_in: 1 week

    expire_in: 1 day

    when: always

    paths:

      - ${BUILD_PATH}
@@ -205,7 +209,7 @@ gkfs:integration: 
## == integration tests for gkfwd ==========

gkfwd:integration:

  stage: test

  image: gekkofs/testing:0.9.4

  image: gekkofs/testing:0.9.5

  interruptible: true

  needs: ['gkfs']

  parallel:
@@ -236,6 +240,7 @@ gkfwd:integration: 
    - cmake --preset ci-coverage

          -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}

          -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/${SUBTEST}.info

          -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external

    ## Since the pipeline recreates the source tree, the access times for .gcno

    ## files are newer than those of .gcda files. This makes gcov emit a

    ## warning for each file which slows it down. Updating the timestamps
@@ -251,7 +256,7 @@ gkfwd:integration: 
          ${BUILD_PATH}/tests/integration/report.xml



  artifacts:

    expire_in: 1 week

    expire_in: 1 day

    when: always

    paths:

      - ${BUILD_PATH}
@@ -262,7 +267,7 @@ gkfwd:integration: 
## == unit tests for gkfs ==================

gkfs:unit:

  stage: test

  image: gekkofs/testing:0.9.4

  image: gekkofs/testing:0.9.5

  needs: ['gkfs']

  script:

    ## Add path to mkfs.kreon
@@ -283,6 +288,7 @@ gkfs:unit: 
    - cmake --preset ci-coverage

          -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}

          -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/unit.info

          -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external

    ## Since the pipeline recreates the source tree, the access times for .gcno

    ## files are newer than those of .gcda files. This makes gcov emit a

    ## warning for each file which slows it down. Updating the timestamps
@@ -291,19 +297,60 @@ gkfs:unit: 
    - cmake --build ${BUILD_PATH} --target coverage-capture



  artifacts:

    expire_in: 1 week

    expire_in: 1 day

    paths:

      - ${BUILD_PATH}

    reports:

      junit: ${BUILD_PATH}/tests/unit/report.xml





## == unit tests for gkfs ==================

gkfs:app:

  stage: test

  image: gekkofs/apps:0.9.5

  needs: ['gkfs']

  script:

    ## Add path to mkfs.kreon

    - export PATH=${PATH}:/usr/local/bin

    ## run actual tests

    - cd ${CI_PROJECT_DIR}

    - ctest --test-dir ${BUILD_PATH}

          -j 1

          -L apps::all

          -VV

          --output-junit ${BUILD_PATH}/tests/apps/report.xml



    ## capture coverage information for this test and write it to

    ## $COVERAGE_PATH/apps.info

    - cd ${CI_PROJECT_DIR}

        # use ccache

    - /usr/sbin/update-ccache-symlinks

    - export PATH="/usr/lib/ccache:$PATH"

    - cmake --preset ci-coverage

          -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}

          -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/apps.info

          -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external

    ## Since the pipeline recreates the source tree, the access times for .gcno

    ## files are newer than those of .gcda files. This makes gcov emit a

    ## warning for each file which slows it down. Updating the timestamps

    ## avoids this

    - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \;

    - cmake --build ${BUILD_PATH} --target coverage-capture



  artifacts:

    expire_in: 1 day

    paths:

      - ${BUILD_PATH}

    reports:

      junit: ${BUILD_PATH}/tests/apps/report.xml





################################################################################

## Generation of documentation

################################################################################

documentation:

  stage: docs

  image: gekkofs/docs:0.9.4

  image: gekkofs/docs:0.9.5

  needs: []

  rules:

    # we only build the documentation automatically if we are on the
@@ -336,7 +383,7 @@ documentation: 
## == coverage baseline ====================

coverage:baseline:

  stage: report

  image: gekkofs/testing:0.9.4

  image: gekkofs/testing:0.9.5

  interruptible: true

  needs: ['gkfs']
@@ -344,13 +391,14 @@ coverage:baseline: 
    ## capture initial coverage information to establish a baseline

    ## and write it to $COVERAGE_PATH/zerocount.info

    # use ccache

    - ccache --zero-stats

    - ccache --zero-stats -M 750MiB --evict-older-than 10d

    - /usr/sbin/update-ccache-symlinks

    - export PATH="/usr/lib/ccache:$PATH"

    - cd ${CI_PROJECT_DIR}

    - cmake --preset ci-coverage

      -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}

      -DCOVERAGE_ZEROCOUNT_TRACEFILE=${COVERAGE_PATH}/zerocount.info

      -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external

    ## Since the pipeline recreates the source tree, the access times for .gcno

    ## files are newer than those of .gcda files. This makes gcov emit a

    ## warning for each file which slows it down. Updating the timestamps
@@ -359,16 +407,16 @@ coverage:baseline: 
    - cmake --build ${BUILD_PATH} --target coverage-zerocount



  artifacts:

    expire_in: 1 week

    expire_in: 1 day

    when: always

    paths:

      - ${COVERAGE_PATH}/zerocount.info



coverage:

  stage: report

  image: gekkofs/testing:0.9.4

  image: gekkofs/testing:0.9.5

  #needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration']

  needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit']

  needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit', 'gkfs:app']

  script:

      # use ccache

    - ccache --zero-stats
@@ -394,16 +442,16 @@ coverage: 
        path: ${COVERAGE_PATH}/coverage-cobertura.xml

    paths:

      - ${COVERAGE_PATH}

    expire_in: 2 weeks

    expire_in: 5 days



cppcheck:

  stage: report

  image: gekkofs/cppcheck:0.9.4-dev

  image: gekkofs/testing:0.9.5

  needs: [ 'gkfs:allintegration', 'gkfs:unit']

  script:

    - cd ${CI_PROJECT_DIR}

    - rm -rf external gkfs

    - cppcheck --xml --enable=warning,style,performance --force ${CI_PROJECT_DIR} 2> cppcheck_out.xml 

    - cppcheck -I/usr/local/include --xml --enable=warning,style,performance --force ${CI_PROJECT_DIR} 2> cppcheck_out.xml 

    - cppcheck-codequality --input-file cppcheck_out.xml --output-file cppcheck.json

  #change paths

  after_script:
@@ -413,28 +461,7 @@ cppcheck: 
      - cppcheck_out.xml

    reports:

      codequality: cppcheck2.json

    expire_in: 2 weeks



sonar:

  stage: report

  image: 

      name: sonarsource/sonar-scanner-cli:11

      entrypoint: [""]

  needs: ['cppcheck']

  rules:

    - if: '$CI_MERGE_REQUEST_EVENT_TYPE == "detached"'

      when: never

    - if: '$CI_MERGE_REQUEST_ID != ""'

      when: manual

      allow_failure: true

  script: 

    - sonar-scanner -Dsonar.host.url="${SONAR_HOST_URL}"

  cache:

    policy: pull-push

    key: "sonar-cache-$CI_COMMIT_REF_SLUG"

    paths:

      - "${SONAR_USER_HOME}/cache"

      - sonar-scanner/

    expire_in: 5 days

  



################################################################################

CHANGELOG.md

+3 −1
Original line number Diff line number Diff line
@@ -28,6 +28,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 
  - Faster initialization relaying on host_files information instead of server RPC ([!242](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/242))

  - Directories shows . and .. to support scandir ([!248](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/248))

  - updated fmt, spdlog and pytest versions ([!249](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/249))

  - Updated Docker images for 0.9.5 ([!238](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/238))

  

### Fixed

  - Dup3 is supported if O_CLOEXEC is not used (i.e. hexdump) ([!228](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/228))
@@ -53,6 +54,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 
- Update capstone to version 6.0.0-Alpha1 and syscall intercept to match ([!215](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/215)).

- Update ARM version and make user library optional in cmake ([!330](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/330))



### Removed

### Fixed



## [0.9.3] - 2024-07

CMake/gkfs-options.cmake

+8 −2
Original line number Diff line number Diff line
@@ -241,6 +241,13 @@ gkfs_define_option( 
    DEFAULT_VALUE ON

)



#build libc

gkfs_define_option(

    GKFS_BUILD_LIBC_INTERCEPTION

    HELP_TEXT "Compile libc interception"

    DEFAULT_VALUE ON

)



# use old resolve function

gkfs_define_option(

    GKFS_USE_LEGACY_PATH_RESOLVE
@@ -287,7 +294,6 @@ gkfs_define_option( 
    DESCRIPTION "Compile with lstat usage in path resolve"

)





################################################################################

# Options and variables that control how GekkoFS behaves internally

################################################################################

CMakeLists.txt

+4 −0
Original line number Diff line number Diff line
@@ -316,6 +316,10 @@ if (GKFS_ENABLE_PROMETHEUS) 
    add_definitions(-DGKFS_ENABLE_PROMETHEUS)

endif ()



if (GKFS_ENABLE_LOCKING)

    add_definitions(-DENABLE_LOCKING)

endif ()



configure_file(include/common/cmake_configure.hpp.in include/common/cmake_configure.hpp)



if (GKFS_ENABLE_CLIENT_LOG)

conftest.py

0 → 100644
+0 −0

Empty file added.