Newer
Older
SCRIPTS_DIR: "${CI_PROJECT_DIR}/scripts"
CI_SCRIPTS_DIR: "${CI_PROJECT_DIR}/scripts/ci"
BUILD_PATH: "${CI_PROJECT_DIR}/gkfs/build"
INSTALL_PATH: "${CI_PROJECT_DIR}/gkfs/install"
INTEGRATION_TESTS_BIN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration"
COVERAGE_PATH: "${CI_PROJECT_DIR}/gkfs/build/.coverage"
PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test"
BATS: "${CI_PROJECT_DIR}/tests/scripts/bats/bin/bats"
LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/deps/install/lib:${CI_PROJECT_DIR}/deps/install/lib64"
GKFS_LOG_LEVEL: "100"
GKFS_DAEMON_LOG_PATH: "${CI_PROJECT_DIR}/logs/daemon.log"
LIBGKFS_LOG: "all"
LIBGKFS_LOG_OUTPUT: "${CI_PROJECT_DIR}/logs/gkfs_client.log"
GIT_SUBMODULE_STRATEGY: recursive
################################################################################
## Validating
################################################################################
- ${SCRIPTS_DIR}/check_format.sh
-s "${CI_PROJECT_DIR}/src"
-i "${CI_PROJECT_DIR}/include"
-v
################################################################################
## Building
################################################################################
gkfs:
stage: build
# Change config.hpp with sed to enable extra features
- sed -i 's/constexpr auto use_atime = false;/constexpr auto use_atime = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"
- sed -i 's/constexpr auto use_ctime = false;/constexpr auto use_ctime = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"
- sed -i 's/constexpr auto use_mtime = false;/constexpr auto use_mtime = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"
- sed -i 's/constexpr auto use_link_cnt = false;/constexpr auto use_link_cnt = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"
- sed -i 's/constexpr auto use_blocks = false;/constexpr auto use_blocks = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"
- mkdir -p ${BUILD_PATH} && cd ${BUILD_PATH}
- cmake
-Wdev
-Wdeprecate
-DCMAKE_BUILD_TYPE=Debug
-DGKFS_ENABLE_CODE_COVERAGE:BOOL=ON
-DGKFS_BUILD_TESTS:BOOL=ON
-DGKFS_INSTALL_TESTS:BOOL=ON
-DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}
-DGKFS_USE_GUIDED_DISTRIBUTION:BOOL=ON
-DGKFS_ENABLE_PARALLAX:BOOL=ON
-DGKFS_ENABLE_ROCKSDB:BOOL=ON
# reduce artifacts size
- ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH}
script:
- mkdir -p ${BUILD_PATH} && cd ${BUILD_PATH}
- cmake
-Wdev
-Wdeprecate
-DCMAKE_BUILD_TYPE=Debug
-DGKFS_INSTALL_TESTS:BOOL=ON
-DGKFS_ENABLE_FORWARDING:BOOL=ON
-DGKFS_ENABLE_AGIOS:BOOL=ON
-DGKFS_ENABLE_PARALLAX:BOOL=OFF
-DGKFS_ENABLE_ROCKSDB:BOOL=ON
-DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}
${CI_PROJECT_DIR}
- make -j$(nproc) install
# reduce artifacts size
- ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH}
- ${BUILD_PATH}
- ${INSTALL_PATH}
################################################################################
## Testing
################################################################################
## == tests for scripts ====================
scripts:
stage: test
needs: []
script:
- mkdir -p ${BUILD_PATH}/tests/scripts
- cd ${BUILD_PATH}/tests/scripts
- ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/dl_dep.sh --formatter junit
- ${BATS} -r ${CI_PROJECT_DIR}/tests/scripts/compile_dep.sh --formatter junit
artifacts:
expire_in: 1 week
reports:
junit: ${BUILD_PATH}/tests/scripts/report.xml
## == integration tests for gkfs ===========
gkfs:integration:
interruptible: true
needs: ['gkfs']
parallel:
matrix:
- SUBTEST: [ data, status, coverage, directories, operations, position, shell ]
## capture coverage information
- cd ${BUILD_PATH}
--root-dir ${CI_PROJECT_DIR}
--build-dir ${BUILD_PATH}
--exclusions "${CI_SCRIPTS_DIR}/.coverage-exclusions"
--log-file "${COVERAGE_PATH}/partial/integration_${SUBTEST}/capture.log"
# fix relative paths so that GitLab can find the correct files
after_script:
- perl -i.orig
-pe 's%file="(.*?)"%file="tests/integration/$1"%;'
-pe 's%(../)+install/share/gkfs/%%g;'
${BUILD_PATH}/tests/integration/report.xml
reports:
junit: ${BUILD_PATH}/tests/integration/report.xml
## == integration tests for gkfwd ==========
gkfwd:integration:
interruptible: true
needs: ['gkfwd']
parallel:
matrix:
- SUBTEST: [ forwarding ]
- cd ${BUILD_PATH}/tests/integration
- ${PYTEST} -v -n $(nproc)
${INTEGRATION_TESTS_BIN_PATH}/${SUBTEST}
--root-dir ${CI_PROJECT_DIR}
--build-dir ${BUILD_PATH}
--exclusions "${CI_SCRIPTS_DIR}/.coverage-exclusions"
--log-file "${COVERAGE_PATH}/partial/integration_${SUBTEST}/capture.log"
# fix relative paths so that GitLab can find the correct files
after_script:
- perl -i.orig
-pe 's%file="(.*?)"%file="tests/integration/$1"%;'
-pe 's%(../)+install/share/gkfs/%%g;'
${BUILD_PATH}/tests/integration/report.xml
- ${BUILD_PATH}
reports:
junit: ${BUILD_PATH}/tests/integration/report.xml
## == unit tests for gkfs ==================
gkfs:unit:
## Add path to mkfs.kreon
- export PATH=${PATH}:/usr/local/bin
## run actual tests
- cd ${BUILD_PATH}/tests/unit
- ctest -j $(nproc) -L unit::all --output-junit report.xml
## capture coverage information
--capture unit
--root-dir ${CI_PROJECT_DIR}
--build-dir ${BUILD_PATH}
--exclusions "${CI_SCRIPTS_DIR}/.coverage-exclusions"
--log-file "${COVERAGE_PATH}/partial/unit/capture.log"
- ${BUILD_PATH}
reports:
junit: ${BUILD_PATH}/tests/unit/report.xml
################################################################################
## Generation of documentation
################################################################################
documentation:
stage: docs
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
needs: []
rules:
# we only build the documentation automatically if we are on the
# `master` branch, but since we also would like to test the documentation
# against our CI, we allow developers to also build it manually
- if: '$CI_MERGE_REQUEST_EVENT_TYPE == "detached"'
when: never
- if: '$CI_MERGE_REQUEST_ID != ""'
when: manual
allow_failure: true
- if: '$CI_COMMIT_REF_SLUG == "master"'
when: always
script:
- mkdir -p ${BUILD_PATH} && cd ${BUILD_PATH}
- cmake
-Wdev
-Wdeprecate
-DCMAKE_BUILD_TYPE=Debug
-DGKFS_BUILD_DOCUMENTATION:BOOL=ON
-DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH}
-DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}
${CI_PROJECT_DIR}
- make docs
artifacts:
paths:
- ${BUILD_PATH}/docs
expire_in: 2 weeks
################################################################################
## Generation of code coverage reports
################################################################################
coverage:
stage: report
needs: [ 'gkfs:integration', 'gkfwd:integration', 'gkfs:unit' ]
script:
- cd ${BUILD_PATH}
## merge the partial coverage files from each test in the pipeline
--merge
--root-dir ${CI_PROJECT_DIR}
--build-dir ${BUILD_PATH}
artifacts:
reports:
cobertura: ${BUILD_PATH}/.coverage/coverage-cobertura.xml
paths:
- ${BUILD_PATH}/.coverage
expire_in: 2 weeks
################################################################################
## Deployment of documentation and reports
################################################################################
#
## for the deploy stage to work as expected, we need to run rsync with the
## appropriate credentials provided by sysadmins. For that, the specific values
## for DEPLOY_KEY_FILE, DEPLOY_USERNAME, DEPLOY_GROUP, DEPLOY_SERVER and
## DEPLOY_PATH must be defined as protected variables.
deploy:
image: bscstorage/deployer
stage: deploy
needs: [ 'documentation' ]
only:
- master
script:
- chmod 400 ${DEPLOY_KEY_FILE}
- rsync -e "ssh -i ${DEPLOY_KEY_FILE}"
-avzr
--delete
--chown=${DEPLOY_USERNAME}:${DEPLOY_GROUP}
${BUILD_PATH}/docs/sphinx/sphinx_docs/
${DEPLOY_USERNAME}@${DEPLOY_SERVER}:${DEPLOY_PATH}