Commit 371d9d29 authored by Ramon Nou's avatar Ramon Nou
Browse files

Update CI and dockers for EC 

Working recovery EC

Polishing touches for Erasure codes
parent 8a5b0190

.gitlab-ci.yml

+27 −10
Original line number Diff line number Diff line
@@ -24,14 +24,14 @@ variables: 
  GIT_SUBMODULE_STRATEGY:       recursive



# base image

image: gekkofs/core:0.9.3-rc1

image: gekkofs/core:0.9.3-exp



################################################################################

## Validating

################################################################################

check format:

  stage: lint

  image: gekkofs/linter:0.9.3-rc1

  image: gekkofs/linter:0.9.3-exp

  needs: []

  script:

    - ${SCRIPTS_DIR}/check_format.sh
@@ -45,7 +45,7 @@ check format: 
################################################################################

gkfs:

  stage: build

  image: gekkofs/deps:0.9.3-rc1

  image: gekkofs/deps:0.9.3-exp

  interruptible: true

  needs: []

  script:
@@ -65,6 +65,23 @@ gkfs: 
      - ${INSTALL_PATH}

    expire_in: 1 week



gkfwd:

  stage: build

  image: gekkofs/deps:0.9.3-exp

  interruptible: true

  needs: []

  script:

    - cmake --preset ci-forwarding-coverage

    - cmake --build ${BUILD_PATH} -j $(nproc) --target install

    # reduce artifacts size

    - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH}

  artifacts:

    paths:

      - ${BUILD_PATH}

      - ${INSTALL_PATH}

    expire_in: 1 week





################################################################################

## Testing

################################################################################
@@ -72,7 +89,7 @@ gkfs: 
## == tests for scripts ====================

scripts:

  stage: test

  image: gekkofs/testing:0.9.3-rc1

  image: gekkofs/testing:0.9.3-exp

  needs: []

  script:

    - mkdir -p ${BUILD_PATH}/tests/scripts
@@ -88,7 +105,7 @@ scripts: 
## == integration tests for gkfs ===========

gkfs:integration:

  stage: test

  image: gekkofs/testing:0.9.3-rc1

  image: gekkofs/testing:0.9.3-exp

  interruptible: true

  needs: ['gkfs']

  parallel:
@@ -137,7 +154,7 @@ gkfs:integration: 
## == integration tests for gkfwd ==========

gkfwd:integration:

  stage: test

  image: gekkofs/testing:0.9.3-rc1

  image: gekkofs/testing:0.9.3-exp

  interruptible: true

  needs: ['gkfs']

  parallel:
@@ -187,7 +204,7 @@ gkfwd:integration: 
## == unit tests for gkfs ==================

gkfs:unit:

  stage: test

  image: gekkofs/testing:0.9.3-rc1

  image: gekkofs/testing:0.9.3-exp

  needs: ['gkfs']

  script:

    ## Add path to mkfs.kreon
@@ -225,7 +242,7 @@ gkfs:unit: 
################################################################################

documentation:

  stage: docs

  image: gekkofs/docs:0.9.3-rc1

  image: gekkofs/docs:0.9.3-exp

  needs: []

  rules:

    # we only build the documentation automatically if we are on the
@@ -255,7 +272,7 @@ documentation: 
## == coverage baseline ====================

coverage:baseline:

  stage: report

  image: gekkofs/testing:0.9.3-rc1

  image: gekkofs/testing:0.9.3-exp

  interruptible: true

  needs: ['gkfs']
@@ -281,7 +298,7 @@ coverage:baseline: 


coverage:

  stage: report

  image: gekkofs/testing:0.9.3-rc1

  image: gekkofs/testing:0.9.3-exp

#  needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration',

#           'gkfs:unit' ]

  needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration']

CHANGELOG.md

+1 −1
Original line number Diff line number Diff line
@@ -55,7 +55,7 @@ replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141) 
replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)).

- Modified write and reads to use a bitset instead of the traditional hash per chunk in the server.

- Added reattemp support in get_fs_config to other servers, when the initial server fails.

- Added support for Erasure codes ([!1xx])

- Added support for Erasure codes ([!168]) using Jerasure lib and adding support for Read error injection.

### New



- Support for client-side per process logging, activated

CMake/gkfs-options.cmake

+9 −1
Original line number Diff line number Diff line
@@ -374,6 +374,13 @@ gkfs_define_option( 
  DESCRIPTION "Use Jerasure for erasure codes reliability"

)



gkfs_define_option(

  GKFS_ENABLE_READ_ERRORS

  HELP_TEXT "Enable Read Errors using replication"

  DEFAULT_VALUE OFF

  DESCRIPTION "Inject read errors"

)



################################################################################

# MSGPack client metrics

################################################################################
@@ -383,3 +390,4 @@ gkfs_define_option( 
    DEFAULT_VALUE OFF

    DESCRIPTION "If GKFS_ENABLE_CLIENT_METRICS is ON, use MSGPack to dump client read/write metrics"

)

CMakePresets.json

+1 −0
Original line number Diff line number Diff line
@@ -79,6 +79,7 @@ 
        "GKFS_RENAME_SUPPORT": true,

        "GKFS_MAX_OPEN_FDS": "10000",

        "GKFS_MAX_INTERNAL_FDS": "1024"

        "GKFS_ENABLE_EC": true

      }

    },

    {

README.md

+4 −1
Original line number Diff line number Diff line
@@ -377,10 +377,13 @@ The number of replicas should go from 0 to the number of servers-1. 
The replication environment variable can be set up for each client, independently.



### Erasure codes

The user can enable resilience with erasure codes with -DGKFS_ENABLE_EC

The user can enable resilience with erasure codes with `-DGKFS_ENABLE_EC`

Using `LIBGKFS_NUM_REPL=<num repl>`, the user can define the number of EC servers.

The total servers available for data will be -> total servervs - num_repl



Additionaly the user can enable the `-DGKFS_ENABLE_READ_ERRORS` to inject a  

50% of read errors into the workflow.



## Acknowledgment



This software was partially supported by the EC H2020 funded NEXTGenIO project (Project ID: 671951, www.nextgenio.eu).