Commit 5c3777f6 authored by Marc Vef's avatar Marc Vef
Browse files

Merge branch 'fmt10' into 'master' 

Update to fmt10

In order to reduce warnings, I propose an update to fmt10.

It builds over the other MR

* fmt10 needs a small change in Hermes (branch fmt10)
* fmt10 needs changes in date library 
* It produces segfaults as we get stuck in a syscall loop
* Removed date library and replaced with a small gmtime replacement without timezone
* Rocksdb updated
* 
* 0.9.3 docker images created

See merge request !172
parents f2092c3b 237f6110

.gitlab-ci.yml

+15 −32
Original line number Diff line number Diff line
@@ -24,14 +24,14 @@ variables: 
  GIT_SUBMODULE_STRATEGY:       recursive



# base image

image: gekkofs/core:0.9.2

image: gekkofs/core:0.9.3



################################################################################

## Validating

################################################################################

check format:

  stage: lint

  image: gekkofs/linter:0.9.2

  image: gekkofs/linter:0.9.3

  needs: []

  script:

    - ${SCRIPTS_DIR}/check_format.sh
@@ -45,7 +45,7 @@ check format: 
################################################################################

gkfs:

  stage: build

  image: gekkofs/deps:0.9.2

  image: gekkofs/deps:0.9.3

  interruptible: true

  needs: []

  script:
@@ -65,23 +65,6 @@ gkfs: 
      - ${INSTALL_PATH}

    expire_in: 1 week



gkfwd:

  stage: build

  image: gekkofs/deps:0.9.2

  interruptible: true

  needs: []

  script:

    - cmake --preset ci-forwarding-coverage

    - cmake --build ${BUILD_PATH} -j $(nproc) --target install

    # reduce artifacts size

    - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH}

  artifacts:

    paths:

      - ${BUILD_PATH}

      - ${INSTALL_PATH}

    expire_in: 1 week





################################################################################

## Testing

################################################################################
@@ -89,7 +72,7 @@ gkfwd: 
## == tests for scripts ====================

scripts:

  stage: test

  image: gekkofs/testing:0.9.2

  image: gekkofs/testing:0.9.3

  needs: []

  script:

    - mkdir -p ${BUILD_PATH}/tests/scripts
@@ -105,7 +88,7 @@ scripts: 
## == integration tests for gkfs ===========

gkfs:integration:

  stage: test

  image: gekkofs/testing:0.9.2

  image: gekkofs/testing:0.9.3

  interruptible: true

  needs: ['gkfs']

  parallel:
@@ -154,14 +137,14 @@ gkfs:integration: 
## == integration tests for gkfwd ==========

gkfwd:integration:

  stage: test

  image: gekkofs/testing:0.9.2

  image: gekkofs/testing:0.9.3

  interruptible: true

  needs: ['gkfwd']

  needs: ['gkfs']

  parallel:

    matrix:

      - SUBTEST: [ forwarding ]

  rules:

    - when: never

 # rules:

 #   - when: never



  script:

    ## run tests
@@ -204,7 +187,7 @@ gkfwd:integration: 
## == unit tests for gkfs ==================

gkfs:unit:

  stage: test

  image: gekkofs/testing:0.9.2

  image: gekkofs/testing:0.9.3

  needs: ['gkfs']

  script:

    ## Add path to mkfs.kreon
@@ -242,7 +225,7 @@ gkfs:unit: 
################################################################################

documentation:

  stage: docs

  image: gekkofs/docs:0.9.2

  image: gekkofs/docs:0.9.3

  needs: []

  rules:

    # we only build the documentation automatically if we are on the
@@ -272,9 +255,9 @@ documentation: 
## == coverage baseline ====================

coverage:baseline:

  stage: report

  image: gekkofs/testing:0.9.2

  image: gekkofs/testing:0.9.3

  interruptible: true

  needs: ['gkfs', 'gkfwd']

  needs: ['gkfs']



  script:

    ## capture initial coverage information to establish a baseline
@@ -298,10 +281,10 @@ coverage:baseline: 


coverage:

  stage: report

  image: gekkofs/testing:0.9.2

  image: gekkofs/testing:0.9.3

#  needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration',

#           'gkfs:unit' ]

  needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit' ]

  needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration']

  script:

    - cd ${CI_PROJECT_DIR}

    - cmake

.gitmodules

+1 −1
Original line number Diff line number Diff line 
[submodule "external/hermes"]

	path = external/hermes

	url = https://github.com/bsc-ssrg/hermes.git

	url = https://github.com/gekkofs/hermes.git

[submodule "tests/scripts/bats"]

	path = tests/scripts/bats

	url = https://github.com/bats-core/bats-core.git

CHANGELOG.md

+3 −0
Original line number Diff line number Diff line
@@ -10,6 +10,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 
replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)).

- Modified write and reads to use a bitset instead of the traditional hash per chunk in the server.

- Added reattemp support in get_fs_config to other servers, when the initial server fails.

- Fused GekkoFWD and GekkoFS. GekkoFWD is enabled with the `--enable-following` in the server configuration and the ENV variable

`LIBGKFS_FORWARDING_MAP_FILE` in the clients. ([!170](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/170)).

- FMT10 and date removal, several dependencies updated. ([!172](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/172)).



### New

### Changed

CMake/gkfs-options.cmake

+0 −8
Original line number Diff line number Diff line
@@ -348,14 +348,6 @@ gkfs_define_option( 
# I/O forwarding

################################################################################



## Forwarding support

gkfs_define_option(

  GKFS_ENABLE_FORWARDING

  HELP_TEXT "Enable I/O forwarding mode"

  DEFAULT_VALUE OFF

  DESCRIPTION "Use ${PROJECT_NAME} as an I/O forwarding layer"

)



## Scheduling in I/O forwarding mode

gkfs_define_option(

  GKFS_ENABLE_AGIOS

CMakeLists.txt

+2 −10
Original line number Diff line number Diff line
@@ -160,10 +160,6 @@ find_package(Margo 0.9.6 REQUIRED) 
message(STATUS "[${PROJECT_NAME}] Checking for syscall_intercept")

find_package(Syscall_intercept REQUIRED)



### Date (required by the libgkfs_intercept.so)

message(STATUS "[${PROJECT_NAME}] Checking for Date")

find_package(Date REQUIRED)



### AGIOS: required for scheduling I/O requests

if (GKFS_ENABLE_AGIOS)

    message(STATUS "[${PROJECT_NAME}] Checking for Agios")
@@ -225,7 +221,7 @@ include_from_source(fmt 
  MESSAGE "[${PROJECT_NAME}] Searching for {fmt}"

  SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/fmt

  GIT_REPOSITORY https://github.com/fmtlib/fmt

  GIT_TAG f94b7364b9409f05207c3af3fa4666730e11a854 # v6.1.2

  GIT_TAG e57ca2e3685b160617d3d95fcd9e789c4e06ca88 # v10.1.0

)



# ensure that fmt is linked as PIC
@@ -338,11 +334,7 @@ if (GKFS_BUILD_TESTS) 
    message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}")



    message(STATUS "[gekkofs] Check for forwarding tests...")

    if (GKFS_ENABLE_FORWARDING)

    set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)")

    else ()

        set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)")

    endif ()

    message(STATUS "[gekkofs] Forwarding tests: ${GKFS_TESTS_FORWARDING}")

    message(STATUS "[gekkofs] Check for guided distributor tests...")

    if (GKFS_USE_GUIDED_DISTRIBUTION)