Commit b0092d69 authored by Ramon Nou's avatar Ramon Nou
Browse files

Resolve "test_concat fails sometimes"

parent a7cab522

.gitlab-ci.yml

+96 −34
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ stages: 
  - deploy



variables:

  VERSION:                      "0.9.6"

  SCRIPTS_DIR:                  "${CI_PROJECT_DIR}/scripts"

  CI_SCRIPTS_DIR:               "${CI_PROJECT_DIR}/scripts/ci"

  BUILD_PATH:                   "${CI_PROJECT_DIR}/gkfs/build"
@@ -23,19 +24,28 @@ variables: 
  LIBGKFS_LOG_OUTPUT:           "${CI_PROJECT_DIR}/logs/gkfs_client.log"

  GIT_SUBMODULE_STRATEGY:       recursive

  CCACHE_DIR:                   "${CI_PROJECT_DIR}/ccache"

#  SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar"  # Defines the location of the analysis task cache

#  GIT_DEPTH: "0"  # Tells git to fetch all the branches of the project, required by the analysis task



  IMAGE_PREFIX:                "${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/"

  #IMAGE_PREFIX:                "gekkofs/"



  CORE:                        "${IMAGE_PREFIX}core:${VERSION}"

  LINTER:                      "${IMAGE_PREFIX}linter:${VERSION}"

  DEPS:                        "${IMAGE_PREFIX}deps:${VERSION}"

  TESTING:                     "${IMAGE_PREFIX}testing:${VERSION}"

  APPS:                        "${IMAGE_PREFIX}apps:${VERSION}"

  JAVA:                        "${IMAGE_PREFIX}java:${VERSION}"

  DOCS:                        "${IMAGE_PREFIX}docs:${VERSION}"

  



# base image

image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/core:0.9.6

image: ${CORE}



################################################################################

## Validating

################################################################################

check format:

  stage: lint

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/linter:0.9.6

  image: ${LINTER}

  needs: []

  script:

    - ${SCRIPTS_DIR}/check_format.sh
@@ -49,7 +59,7 @@ check format: 
################################################################################

gkfs:

  stage: build

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/deps:0.9.6

  image: ${DEPS}

  interruptible: true

  needs: []

  script:
@@ -88,7 +98,7 @@ gkfs: 
## == tests for scripts ====================

scripts:

  stage: test

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6

  image: ${TESTING}

  needs: []

  script:

    - mkdir -p ${BUILD_PATH}/tests/scripts
@@ -100,10 +110,10 @@ scripts: 
    reports:

      junit: ${BUILD_PATH}/tests/scripts/report.xml



## == integration tests for gkfs ===========

gkfs:allintegration:

## == integration tests for gkfs (group 1: I/O, FUSE, shell) ================

gkfs:integration-1:

  stage: test

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6

  image: ${TESTING}

  interruptible: true

  needs: ['gkfs']
@@ -113,32 +123,30 @@ gkfs:allintegration: 
    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - ${PYTEST} -v -n $(nproc)

          ${INTEGRATION_TESTS_BIN_PATH}/

          ${INTEGRATION_TESTS_BIN_PATH}/data

          ${INTEGRATION_TESTS_BIN_PATH}/fuse

          ${INTEGRATION_TESTS_BIN_PATH}/shell

          ${INTEGRATION_TESTS_BIN_PATH}/compatibility

          ${INTEGRATION_TESTS_BIN_PATH}/startup

          ${INTEGRATION_TESTS_BIN_PATH}/error_handling

          --basetemp=${BUILD_PATH}/tests/run/

          --junit-xml=report.xml

          --junit-xml=report-1.xml



    ## capture coverage information for this test and write it to

    ## $COVERAGE_PATH/$SUBTEST.info

    ## capture coverage information

    - cd ${CI_PROJECT_DIR}

    # use ccache

    - /usr/sbin/update-ccache-symlinks

    - export PATH="/usr/lib/ccache:$PATH"

    - cmake --preset ci-coverage

          -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}

          -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration.info

    ## Since the pipeline recreates the source tree, the access times for .gcno

    ## files are newer than those of .gcda files. This makes gcov emit a

    ## warning for each file which slows it down. Updating the timestamps

    ## avoids this

          -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration1.info

    - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \;

    - cmake --build ${BUILD_PATH} --target coverage-capture



  # fix relative paths so that GitLab can find the correct files

  after_script:

    - perl -i.orig

          -pe 's%file="(.*?)"%file="tests/integration/$1"%g;'

          -pe 's%(../)+install/share/gkfs/%%g;'

          ${BUILD_PATH}/tests/integration/report.xml

          ${BUILD_PATH}/tests/integration/report-1.xml



  artifacts:

    expire_in: 1 day
@@ -146,12 +154,62 @@ gkfs:allintegration: 
    paths:

      - ${BUILD_PATH}

    reports:

      junit: ${BUILD_PATH}/tests/integration/report.xml

      junit: ${BUILD_PATH}/tests/integration/report-1.xml





gkfs:integration-2:

  stage: test

  image: ${TESTING}

  interruptible: true

  needs: ['gkfs']



  script:

    ## run tests

    - export PATH=${PATH}:/usr/local/bin

    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - ${PYTEST} -v -n $(nproc)

          ${INTEGRATION_TESTS_BIN_PATH}/syscalls

          ${INTEGRATION_TESTS_BIN_PATH}/directories

          ${INTEGRATION_TESTS_BIN_PATH}/operations

          ${INTEGRATION_TESTS_BIN_PATH}/rename

          ${INTEGRATION_TESTS_BIN_PATH}/position

          ${INTEGRATION_TESTS_BIN_PATH}/status

          ${INTEGRATION_TESTS_BIN_PATH}/malleability

          ${INTEGRATION_TESTS_BIN_PATH}/forwarding

          ${INTEGRATION_TESTS_BIN_PATH}/concurrency

          ${INTEGRATION_TESTS_BIN_PATH}/resilience

          --basetemp=${BUILD_PATH}/tests/run/

          --junit-xml=report-2.xml



    ## capture coverage information

    - cd ${CI_PROJECT_DIR}

    - /usr/sbin/update-ccache-symlinks

    - export PATH="/usr/lib/ccache:$PATH"

    - cmake --preset ci-coverage

          -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}

          -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration2.info

    - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \;

    - cmake --build ${BUILD_PATH} --target coverage-capture



  after_script:

    - perl -i.orig

          -pe 's%file="(.*?)"%file="tests/integration/$1"%g;'

          -pe 's%(../)+install/share/gkfs/%%g;'

          ${BUILD_PATH}/tests/integration/report-2.xml



  artifacts:

    expire_in: 1 day

    when: always

    paths:

      - ${BUILD_PATH}

    reports:

      junit: ${BUILD_PATH}/tests/integration/report-2.xml



## == integration tests for gkfs ===========

gkfs:integration:

  stage: test

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6

  image: ${TESTING}

  interruptible: true

  needs: ['gkfs']  # we need to remove gkfs dependencies on manual

  parallel:
@@ -209,7 +267,7 @@ gkfs:integration: 
## == integration tests for gkfwd ==========

gkfwd:integration:

  stage: test

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6

  image: ${TESTING}

  interruptible: true

  needs: ['gkfs']

  parallel:
@@ -267,7 +325,8 @@ gkfwd:integration: 
## == unit tests for gkfs ==================

gkfs:unit:

  stage: test

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6

  image: ${TESTING}

  interruptible: true

  needs: ['gkfs']

  script:

    ## Add path to mkfs.kreon
@@ -307,7 +366,8 @@ gkfs:unit: 
## == unit tests for gkfs ==================

gkfs:app:

  stage: test

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/apps:0.9.6

  image: ${APPS}

  interruptible: true

  needs: ['gkfs']

  script:

    ## Add path to mkfs.kreon
@@ -348,7 +408,8 @@ gkfs:app: 
## == java tests for gkfs ==================

gkfs:java:

  stage: test

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/java:0.9.6

  image: ${JAVA}

  interruptible: true

  needs: ['gkfs']

  script:

    ## Add path to mkfs.kreon
@@ -389,7 +450,8 @@ gkfs:java: 
## == python tests for gkfs ==================

gkfs:python:

  stage: test

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6

  image: ${TESTING}

  interruptible: true

  needs: ['gkfs']

  script:

    ## Add path to mkfs.kreon
@@ -432,7 +494,7 @@ gkfs:python: 
################################################################################

documentation:

  stage: docs

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/docs:0.9.6

  image: ${DOCS}

  needs: []

  rules:

    # we only build the documentation automatically if we are on the
@@ -465,7 +527,7 @@ documentation: 
## == coverage baseline ====================

coverage:baseline:

  stage: report

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6

  image: ${TESTING}

  interruptible: true

  needs: ['gkfs']
@@ -496,9 +558,9 @@ coverage:baseline: 


coverage:

  stage: report

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6

  image: ${TESTING}

  #needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration']

  needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ]

  needs: [ 'coverage:baseline', 'gkfs:integration-1', 'gkfs:integration-2', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ]

  script:

      # use ccache

    - ccache --zero-stats
@@ -528,7 +590,7 @@ coverage: 


cppcheck:

  stage: report

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6

  image: ${TESTING}

  needs: [ 'gkfs']

  script:

    - cd ${CI_PROJECT_DIR}
@@ -554,7 +616,7 @@ cppcheck: 
## for DEPLOY_KEY_FILE, DEPLOY_USERNAME, DEPLOY_GROUP, DEPLOY_SERVER and

## DEPLOY_PATH must be defined as protected variables.

deploy:

  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/bscstorage/deployer

  image: bscstorage/deployer

  stage: deploy

  needs: [ 'documentation' ]

  only:

docker/0.9.6/deps/Dockerfile

+2 −1
Original line number Diff line number Diff line
@@ -23,7 +23,8 @@ RUN apt-get update && \ 
  python3-setuptools \

  libnuma-dev libyaml-dev libcurl4-openssl-dev \

  libfuse3-dev fuse3 \

  procps && \

  procps \

  kmod && \

  rm -rf /var/lib/apt/lists/* && \

  apt-get clean && apt-get autoclean

include/client/open_file_map.hpp

+1 −1
Original line number Diff line number Diff line
@@ -66,7 +66,7 @@ enum class OpenFile_flags { 
    flag_count // this is purely used as a size variable of this enum class

};



enum class FileType { regular, directory };

enum class FileType { regular, directory, symlink };



class OpenFile {

protected:

include/client/rpc/forward_metadata.hpp

+4 −3
Original line number Diff line number Diff line
@@ -101,8 +101,8 @@ forward_get_metadentry_size(const std::string& path, const int copy); 
std::pair<int, std::shared_ptr<gkfs::filemap::OpenDir>>

forward_get_dirents(const std::string& path);



std::pair<int, std::unique_ptr<std::vector<

                       std::tuple<const std::string, bool, size_t, time_t>>>>

std::pair<int, std::unique_ptr<std::vector<std::tuple<

                       const std::string, unsigned char, size_t, time_t>>>>

forward_get_dirents_single(const std::string& path, int server,

                           const std::string& start_key = "",

                           bool get_all = true);
@@ -143,7 +143,8 @@ forward_read_inline(const std::string& path, void* buf, off64_t offset, 
                    size_t read_size);



std::tuple<int,

           std::vector<std::tuple<const std::string, bool, size_t, time_t>>,

           std::vector<std::tuple<const std::string, unsigned char, size_t,

                                  time_t>>,

           uint64_t, std::string>

forward_get_dirents_filtered(const std::string& path, int server,

                             const std::string& start_key,

include/client/rpc/forward_metadata_proxy.hpp

+2 −2
Original line number Diff line number Diff line
@@ -52,8 +52,8 @@ forward_update_metadentry_size_proxy(const std::string& path, const size_t size, 
std::pair<int, off64_t>

forward_get_metadentry_size_proxy(const std::string& path);



std::pair<int, std::unique_ptr<std::vector<

                       std::tuple<const std::string, bool, size_t, time_t>>>>

std::pair<int, std::unique_ptr<std::vector<std::tuple<

                       const std::string, unsigned char, size_t, time_t>>>>

forward_get_dirents_single_proxy_v2(const std::string& path, int server);



} // namespace gkfs::rpc